def delete_profile(profile_id, uci):
""" delete a specific profile """
profiles = get_profiles()
try:
profile = next(profile for profile in profiles["profiles"]
if profile["id"] == profile_id)
associated_networks = [
network["id"] for network in networks.get_networks(uci)
if network["profileId"] == profile["id"]
]
reload_dropbear, reload_firewall = (False, False)
for network_id in associated_networks:
reload_dropbear, reload_firewall = update_profile_network(
uci, network_id, profile, ALL_OFF_PROFILE)
uci.set_option(ADMIN_PKG, network_id, "profile_id", "")
if reload_dropbear:
run(["/etc/init.d/dropbear", "reload"], check=False)
if reload_firewall:
run(["/etc/init.d/firewall", "reload"], check=False)
uci.persist(ADMIN_PKG)
PROFILES_APP.profiles["profiles"].remove(profile)
persist_profiles()
response.status = 204
return ""
except StopIteration:
response.status = 404
return "Invalid Profile ID"
def load_train_models(state):
if state.train_nets_type == 'unknown_init':
model, = networks.get_networks(state, N=1)
return [model for _ in range(state.local_n_nets)]
elif state.train_nets_type == 'known_init':
return networks.get_networks(state, N=state.local_n_nets)
elif state.train_nets_type == 'loaded':
models = networks.get_networks(state, N=state.local_n_nets)
with state.pretend(phase='train'): # in case test_nets_type == same_as_train
model_dir = state.get_model_dir()
start_idx = state.world_rank * state.local_n_nets
for n, model in enumerate(models, start_idx):
model_path = os.path.join(model_dir, 'net_{:04d}'.format(n))
model.load_state_dict(torch.load(model_path, map_location=state.device))
logging.info('Loaded checkpoints [{} ... {}) from {}'.format(
start_idx, start_idx + state.local_n_nets, model_dir))
return models
else:
raise ValueError("train_nets_type: {}".format(state.train_nets_type))
def distillation(exp_dir):
for rst_i in tqdm(range(TOTAL_RESTARTS), desc='distillation'):
res = {}
for arch_nm in ['LinearNet', 'NonLinearNet', 'MoreNonLinearNet']:
state = deepcopy(BASE_OPTIONS)
state.update({
'arch': arch_nm,
'distill_epochs': DISTILL_EPOCHS,
'distill_steps': DISTILL_STEPS,
'epochs': EPOCHS,
})
state = State(**state)
state['models'] = get_networks(state, N=3)
state['test_models'] = get_networks(state, N=1)
st = time()
steps, losses = train_distilled_image.distill(state, state.models)
res[arch_nm] = ResEl(state, steps, losses, time() - st)
with open(os.path.join(exp_dir, f'res_{rst_i}.pk'), 'wb') as f:
pk.dump(res, f)
def distill_with_param(exp_dir, param_vars, param_initer):
for rst_i in tqdm(range(TOTAL_RESTARTS), desc='distill_with_param'):
res = {}
for arch_nm in ['LinearNet', 'NonLinearNet', 'MoreNonLinearNet']:
res[arch_nm] = {}
for param in param_vars:
state = deepcopy(BASE_OPTIONS)
state['arch'] = arch_nm
state = param_initer(state, param)
state['test_models'] = get_networks(state, N=1)
st = time()
steps, losses = train_distilled_image.distill(
state, state.models)
res[arch_nm][param] = ResEl(state, steps, losses, time() - st)
with open(os.path.join(exp_dir, f'res_{rst_i}.pk'), 'wb') as f:
pk.dump(res, f)
def _test_params_invariance(self, state):
models = networks.get_networks(state, 1)
trainer = Trainer(state, models)
model = trainer.models[0]
ref_w = model.get_param(clone=True)
rdata, rlabel = next(iter(state.train_loader))
rdata = rdata.to(state.device, non_blocking=True)
rlabel = rlabel.to(state.device, non_blocking=True)
model.train()
steps = trainer.get_steps()
l, saved = trainer.forward(model, rdata, rlabel, steps)
self.assertTrue(torch.equal(ref_w, model.get_param()))
trainer.backward(model, rdata, rlabel, steps, saved)
self.assertTrue(torch.equal(ref_w, model.get_param()))
def update_profile(profile_id, uci):
""" update a specific profile """
try:
profiles = get_profiles()
profile = next(profile for profile in profiles["profiles"]
if profile["id"] == profile_id)
updated_profile = dict(request.json)
if not validate_profile(updated_profile, uci):
response.status = 400
return "Empty or invalid content"
associated_networks = [
network["id"] for network in networks.get_networks(uci)
if network["profileId"] == profile_id
]
reload_dropbear, reload_firewall = (False, False)
for network_id in associated_networks:
reload_dropbear, reload_firewall = update_profile_network(
uci, network_id, profile, updated_profile)
if reload_dropbear:
run(["/etc/init.d/dropbear", "reload"], check=False)
if reload_firewall:
run(["/etc/init.d/firewall", "reload"], check=False)
profile["name"] = updated_profile["name"]
profile["type"] = updated_profile["type"]
profile["ssh"] = updated_profile["ssh"]
profile["deviceBlocking"] = updated_profile["deviceBlocking"]
profile["siteBlocking"] = updated_profile["siteBlocking"]
profile["siteBlocking"]["sites"] = sorted(
set(updated_profile["siteBlocking"]["sites"]))
profile["deviceAccess"] = updated_profile["deviceAccess"]
profile["deviceAccess"]["devices"] = sorted(
set(updated_profile["deviceAccess"]["devices"]))
persist_profiles()
return profile
except StopIteration:
response.status = 404
return "Invalid id"
except (JSONDecodeError, KeyError, TypeError):
response.status = 400
return "Invalid content"
def main():
args = distillation_parser().parse_args()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
dataset, channels, squared_dim = select_dataset(
args.dataset, './data/', args.training_data_batch_size, True)
nets_args = {}
for net_name in args.networks:
if net_name in TRAINABLE_NETWORKS:
nets_args[net_name] = {
'input_channels': channels,
'num_classes': args.n_classes,
'input_dims': squared_dim
}
networks = get_networks(args.networks, args.pretrained, nets_args)
loss_fn = nn.CrossEntropyLoss()
weights_init_fn = create_weights_init_fn(nn.init.xavier_normal_, gain=1.0)
distillation_trainer = DistillationTrainer(
networks,
device,
args.distill_epochs,
args.epochs,
args.distilled_batches,
args.n_classes,
args.examples_per_class, (channels, squared_dim, squared_dim),
args.distill_lr,
args.optimizer_name,
args.alpha,
loss_fn,
weights_init_fn,
dataset,
save_log_fn=save_distilled_log_data)
distilled_data = distillation_trainer.distill()
save_distilled_data(distilled_data, './output/')
def main(state):
logging.info('mode: {}, phase: {}'.format(state.mode, state.phase))
if state.mode == 'train':
model_dir = state.get_model_dir()
utils.mkdir(model_dir)
start_idx = cur_idx = state.world_rank * state.local_n_nets
end_idx = start_idx + state.local_n_nets
if state.train_nets_type == 'loaded':
logging.info('Loading checkpoints [{} ... {}) from {}'.format(
start_idx, end_idx, model_dir))
else:
logging.info('Save checkpoints [{} ... {}) to {}'.format(
start_idx, end_idx, model_dir))
queue_size = 10 # heuristics
while cur_idx < end_idx:
next_cur_idx = min(end_idx, cur_idx + queue_size)
models = networks.get_networks(state, N=(next_cur_idx - cur_idx))
for n, model in enumerate(models, start=cur_idx):
if n == start_idx:
print_network(model)
logging.info('Train network {:04d}'.format(n))
if state.train_nets_type == 'loaded':
model_path = os.path.join(model_dir,
'net_{:04d}'.format(n))
model.load_state_dict(
torch.load(model_path, map_location=state.device))
logging.info('Loaded from {}'.format(model_path))
optimizer = optim.Adam(model.parameters(),
lr=state.lr,
betas=(0.5, 0.999))
scheduler = optim.lr_scheduler.StepLR(
optimizer,
step_size=state.decay_epochs,
gamma=state.decay_factor)
for epoch in range(state.epochs):
scheduler.step()
train(state, model, epoch, optimizer)
model_path = os.path.join(model_dir, 'net_{:04d}'.format(n))
if state.train_nets_type == 'loaded':
model_path += '_{}'.format(state.dataset)
torch.save(model.state_dict(), model_path)
acc, loss = evaluate_models(state, models, test_all=True)
desc = 'Test networks in [{} ... {})'.format(cur_idx, next_cur_idx)
logging.info(
'{}:\tTest Acc: {: >5.2f}%\tTest Loss: {: >7.4f}'.format(
desc,
acc.mean() * 100, loss.mean()))
cur_idx = next_cur_idx
elif state.mode in ['distill_basic', 'distill_attack', 'distill_adapt']:
# train models
def load_train_models():
if state.train_nets_type == 'unknown_init':
model, = networks.get_networks(state, N=1)
return [model for _ in range(state.local_n_nets)]
elif state.train_nets_type == 'known_init':
return networks.get_networks(state, N=state.local_n_nets)
elif state.train_nets_type == 'loaded':
models = networks.get_networks(state, N=state.local_n_nets)
with state.pretend(
phase='train'
): # in case test_nets_type == same_as_train
model_dir = state.get_model_dir()
start_idx = state.world_rank * state.local_n_nets
for n, model in enumerate(models, start_idx):
model_path = os.path.join(model_dir,
'net_{:04d}'.format(n))
model.load_state_dict(
torch.load(model_path, map_location=state.device))
logging.info('Loaded checkpoints [{} ... {}) from {}'.format(
start_idx, start_idx + state.local_n_nets, model_dir))
return models
else:
raise ValueError("train_nets_type: {}".format(
state.train_nets_type))
# only construct when in training mode or test_nets_type == same_as_train
if state.phase == 'train' or state.test_nets_type == 'same_as_train':
state.models = load_train_models()
# test models
if state.test_nets_type == 'unknown_init':
test_model, = networks.get_networks(state, N=1)
state.test_models = [
test_model for _ in range(state.local_test_n_nets)
]
elif state.test_nets_type == 'same_as_train':
assert state.test_n_nets == state.n_nets, \
"test_nets_type=same_as_train, expect test_n_nets=n_nets"
state.test_models = state.models
elif state.test_nets_type == 'loaded':
state.test_models = networks.get_networks(
state, N=state.local_test_n_nets)
with state.pretend(phase='test'):
model_dir = state.get_model_dir() # get test models
start_idx = state.world_rank * state.local_test_n_nets
for n, test_model in enumerate(state.test_models, start_idx):
model_path = os.path.join(model_dir, 'net_{:04d}'.format(n))
test_model.load_state_dict(
torch.load(model_path, map_location=state.device))
logging.info(
'Loaded held-out checkpoints [{} ... {}) from {}'.format(
start_idx, start_idx + state.local_test_n_nets, model_dir))
if state.phase == 'train':
logging.info('Train {} steps iterated for {} epochs'.format(
state.distill_steps, state.distill_epochs))
steps = train_distilled_image.distill(state, state.models)
evaluate_steps(state,
steps,
'distilled with {} steps and {} epochs'.format(
state.distill_steps, state.distill_epochs),
test_all=True)
elif state.phase == 'test':
logging.info('')
logging.info(
('Test:\n'
'\ttest_distilled_images:\t{}\n'
'\ttest_distilled_lrs:\t{}\n'
'\ttest_distill_epochs:\t{}\n'
'\ttest_optmize_n_runs:\t{}\n'
'\ttest_optmize_n_nets:\t{}\n'
'\t{} time(s)').format(state.test_distilled_images,
' '.join(state.test_distilled_lrs),
state.test_distill_epochs,
state.test_optimize_n_runs,
state.test_optimize_n_nets,
state.test_n_runs))
logging.info('')
loaded_steps = load_results(state, device=state.device) # loaded
if state.test_distilled_images == 'loaded':
unique_data_label = [
s[:-1] for s in loaded_steps[:state.distill_steps]
]
def get_data_label(state):
return [
x for _ in range(state.distill_epochs)
for x in unique_data_label
]
elif state.test_distilled_images == 'random_train':
get_data_label = utils.baselines.random_train
elif state.test_distilled_images == 'average_train':
avg_images = None
def get_data_label(state):
nonlocal avg_images
if avg_images is None:
avg_images = utils.baselines.average_train(state)
return avg_images
elif state.test_distilled_images == 'kmeans_train':
get_data_label = utils.baselines.kmeans_train
else:
raise NotImplementedError('test_distilled_images: {}'.format(
state.test_distilled_images))
# get lrs
# allow for passing multiple options
lr_meth = state.test_distilled_lrs[0]
if lr_meth == 'nearest_neighbor':
assert state.mode == 'distill_basic', 'nearest_neighbor test only supports distill_basic'
assert state.test_distill_epochs is None, 'nearest_neighbor test expects unset test_distill_epochs'
assert state.test_optimize_n_runs is None, 'nearest_neighbor test expects unset test_optimize_n_runs'
k = int(state.test_distilled_lrs[1])
p = float(state.test_distilled_lrs[2])
class TestRunner(object):
def __init__(self, state):
self.state = state
def run(self, test_idx, test_at_steps=None):
assert test_at_steps is None
logging.info(
'Test #{} nearest neighbor classification with k={} and {}-norm'
.format(test_idx, k, p))
state = self.state
with state.pretend(distill_epochs=1):
ref_data_label = tuple(get_data_label(state))
ref_flat_data = torch.cat(
[d for d, _ in ref_data_label], 0).flatten(1)
ref_label = torch.cat([l for _, l in ref_data_label],
0)
assert k <= ref_label.size(0), (
'k={} is greater than the number of data {}. '
'Set k to the latter').format(
k, ref_label.size(0))
total = np.array(0, dtype=np.int64)
corrects = np.array(0, dtype=np.int64)
for data, target in state.test_loader:
data = data.to(state.device, non_blocking=True)
target = target.to(state.device, non_blocking=True)
dists = torch.norm(data.flatten(1)[:, None, ...] -
ref_flat_data,
dim=2,
p=p)
if k == 1:
argmin_dist = dists.argmin(dim=1)
pred = ref_label[argmin_dist]
del argmin_dist
else:
_, argmink_dist = torch.topk(dists,
k,
dim=1,
largest=False,
sorted=False)
labels = ref_label[argmink_dist]
counts = [
torch.bincount(l,
minlength=state.num_classes)
for l in labels
]
counts = torch.stack(counts, 0)
pred = counts.argmax(dim=1)
del argmink_dist, labels, counts
corrects += (pred == target).sum().item()
total += data.size(0)
at_steps = torch.ones(1,
dtype=torch.long,
device=state.device)
acc = torch.as_tensor(corrects / total,
device=state.device).view(
1, 1) # STEP x MODEL
loss = torch.full_like(acc, utils.nan) # STEP x MODEL
return (at_steps, acc, loss)
def num_steps(self):
return 1
else:
if lr_meth == 'loaded':
assert state.test_distill_epochs is None
def get_lrs(state):
return tuple(s[-1] for s in loaded_steps)
elif lr_meth == 'fix':
val = float(state.test_distilled_lrs[1])
def get_lrs(state):
n_steps = state.distill_steps * state.distill_epochs
return torch.full((n_steps, ),
val,
device=state.device).unbind()
else:
raise NotImplementedError(
'test_distilled_lrs first: {}'.format(lr_meth))
if state.test_optimize_n_runs is None:
class StepCollection(object):
def __init__(self, state):
self.state = state
def __getitem__(self, test_idx):
steps = []
for (data,
label), lr in zip(get_data_label(self.state),
get_lrs(self.state)):
steps.append((data, label, lr))
return steps
else:
assert state.test_optimize_n_runs >= state.test_n_runs
class StepCollection(object):
@functools.total_ordering
class Step(object):
def __init__(self, step, acc):
self.step = step
self.acc = acc
def __lt__(self, other):
return self.acc < other.acc
def __eq__(self, other):
return self.acc == other.acc
def __init__(self, state):
self.state = state
self.good_steps = [] # min heap
logging.info('Start optimizing evaluated steps...')
for run_idx in range(state.test_optimize_n_runs):
if state.test_nets_type == 'unknown_init':
subtest_nets = [
state.test_models[0] for _ in range(
state.test_optimize_n_nets)
]
else:
with state.pretend(local_n_nets=state.
test_optimize_n_nets):
with utils.logging.disable(
logging.INFO):
subtest_nets = load_train_models()
with state.pretend(
test_models=subtest_nets,
test_loader=state.train_loader):
steps = []
for (data, label), lr in zip(
get_data_label(self.state),
get_lrs(self.state)):
steps.append((data, label, lr))
res = evaluate_steps(
state,
steps,
'',
'',
test_all=False,
test_at_steps=[len(steps)],
log_results=False)
acc = self.acc(res)
elem = StepCollection.Step(steps, acc)
if len(self.good_steps
) < state.test_n_runs:
heapq.heappush(self.good_steps, elem)
else:
heapq.heappushpop(
self.good_steps, elem)
logging.info((
'\tOptimize run {:> 3}:\tAcc on training set {: >5.2f}%'
'\tBoundary Acc {: >5.2f}%').format(
run_idx, acc * 100,
self.good_steps[0].acc * 100))
logging.info('done')
def acc(self, res):
state = self.state
if state.mode != 'distill_attack':
return res[1].mean().item()
else:
return res[1][:, 1].mean().item()
def __getitem__(self, test_idx):
return self.good_steps[test_idx].step
class TestRunner(object): # noqa F811
def __init__(self, state):
self.state = state
if state.test_distill_epochs is None:
self.test_distill_epochs = state.distill_epochs
else:
self.test_distill_epochs = state.test_distill_epochs
with state.pretend(
distill_epochs=self.test_distill_epochs):
self.stepss = StepCollection(state)
def run(self, test_idx, test_at_steps=None):
with self.state.pretend(
distill_epochs=self.test_distill_epochs):
steps = self.stepss[test_idx] # before seeding!
with self.seed(self.state.seed + 1 + test_idx):
return evaluate_steps(
self.state,
steps,
'Test #{}'.format(test_idx),
'({}) images & ({}) lrs'.format(
self.state.test_distilled_images,
' '.join(state.test_distilled_lrs)),
test_all=True,
test_at_steps=test_at_steps)
@contextmanager
def seed(self, seed):
cpu_rng = torch.get_rng_state()
cuda_rng = torch.cuda.get_rng_state(self.state.device)
torch.random.default_generator.manual_seed(seed)
torch.cuda.manual_seed(seed)
yield
torch.set_rng_state(cpu_rng)
torch.cuda.set_rng_state(cuda_rng, self.state.device)
def num_steps(self):
return self.state.distill_steps * self.test_distill_epochs
# run tests
test_runner = TestRunner(state)
cache_init_res = state.test_nets_type != 'unknown_init'
ress = []
for idx in range(state.test_n_runs):
if cache_init_res and idx > 0:
test_at_steps = [-1]
else:
test_at_steps = None
res = test_runner.run(idx, test_at_steps)
if cache_init_res:
if idx == 0:
assert res[0][0].item() == 0
else:
cached = ress[0]
res = (cached[0], torch.cat([cached[1][:1], res[1]],
0),
torch.cat([cached[2][:1], res[2]], 0))
ress.append(res)
# See NOTE [ Evaluation Result Format ] for output format
if state.test_n_runs == 1:
results = ress[0]
else:
results = (
ress[0][0], # at_steps
torch.cat([v[1] for v in ress], 1), # accs
torch.cat([v[2] for v in ress], 1), # losses
)
logging.info('')
# Use dummy learning rates to print summary
steps = [(None, None, np.array(utils.nan))
for _ in range(test_runner.num_steps())]
test_desc = '({}) images & ({}) lrs'.format(
state.test_distilled_images,
' '.join(state.test_distilled_lrs))
logging.info(
format_stepwise_results(state, steps,
'Summary with ' + test_desc, results))
save_test_results(state, results)
logging.info('')
else:
raise ValueError('phase: {}'.format(state.phase))
else:
raise NotImplementedError('unknown mode: {}'.format(state.mode))
def _test_backward(self,
state,
eps=2e-8,
atol=1e-5,
rtol=1e-3,
max_num_per_param=5):
@contextlib.contextmanager
def double_prec():
saved_dtype = torch.get_default_dtype()
torch.set_default_dtype(torch.double)
yield
torch.set_default_dtype(saved_dtype)
with double_prec():
models = [
m.to(torch.double) for m in networks.get_networks(state, 1)
]
trainer = Trainer(state, models)
model = trainer.models[0]
rdata, rlabel = next(iter(state.train_loader))
rdata = rdata.to(state.device, torch.double, non_blocking=True)
rlabel = rlabel.to(state.device, non_blocking=True)
steps = trainer.get_steps()
l, saved = trainer.forward(model, rdata, rlabel, steps)
grad_info = trainer.backward(model, rdata, rlabel, steps, saved)
trainer.accumulate_grad([grad_info])
with torch.no_grad():
for p_idx, p in enumerate(trainer.params):
pdata = p.data
N = p.numel()
for flat_i in np.random.choice(N,
min(N, max_num_per_param),
replace=False):
i = []
for s in reversed(p.size()):
i.insert(0, flat_i % s)
flat_i //= s
i = tuple(i)
ag = p.grad[i].item()
orig = pdata[i].item()
pdata[i] -= eps
steps = trainer.get_steps()
lm, _ = trainer.forward(model, rdata, rlabel, steps)
pdata[i] += eps * 2
steps = trainer.get_steps()
lp, _ = trainer.forward(model, rdata, rlabel, steps)
ng = (lp - lm).item() / (2 * eps)
pdata[i] = orig
rel_err = abs(ag - ng) / (atol + rtol * abs(ng))
info_msg = "testing param {} with shape [{}] at ({}):\trel_err={:.4f}\t" \
"analytical={:+.6f}\tnumerical={:+.6f}".format(
p_idx, format_intlist(p.size()),
format_intlist(i), rel_err, ag, ng)
if unittest_verbosity() > 0:
print(info_msg)
self.assertTrue(rel_err <= 1,
"gradcheck failed when " + info_msg)
def inn_models_initer(state, inn_models):
state.update({'distill_epochs': 1, 'distill_steps': 10})
state = State(**state)
state['models'] = get_networks(state, N=inn_models)
return state
def distill_steps_initer(state, distill_steps):
state.update({'distill_epochs': 1, 'distill_steps': distill_steps})
state = State(**state)
state['models'] = get_networks(state, N=1)
return state
