def run_restrictedImgnetHypo(auto_var):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
_ = set_random_seed(auto_var)
norm = auto_var.get_var("norm")
trn_ds, tst_ds = auto_var.get_var("dataset")
lbl_enc = None # TODO
auto_var.set_intermidiate_variable("lbl_enc", lbl_enc)
n_classes = len(trn_ds.classes)
result = {}
mock_trnX = np.concatenate([trn_ds[0][0], trn_ds[1][0]], axis=0)
trny = np.array(trn_ds.targets)
tsty = np.array(tst_ds.targets)
#multigpu = True if torch.cuda.device_count() > 1 else False
multigpu = False
model = auto_var.get_var("model", trnX=trnX, trny=trny, multigpu=multigpu, n_channels=n_channels)
model.tst_ds = (tstX, tsty)
with Stopwatch("Fitting Model"):
history = model.fit(trnX, trny)
result['trn_acc'] = (model.predict_ds(trn_ds) == trny).mean()
result['tst_acc'] = (model.predict_ds(tst_ds) == tsty).mean()
print(f"train acc: {result['trn_acc']}")
print(f"test acc: {result['tst_acc']}")
attack_model = auto_var.get_var("attack", model=model, n_classes=n_classes,
clip_min=0, clip_max=1,)
with Stopwatch("Attacking Train"):
adv_trnX = attack_model.perturb_ds(trn_ds)
with Stopwatch("Attacking Test"):
adv_tstX = attack_model.perturb_ds(tst_ds)
result['adv_trn_acc'] = (model.predict(adv_trnX) == trny).mean()
result['adv_tst_acc'] = (model.predict(adv_tstX) == tsty).mean()
print(f"adv trn acc: {result['adv_trn_acc']}")
print(f"adv tst acc: {result['adv_tst_acc']}")
del attack_model
with Stopwatch("Estimating trn Lip"):
trn_lip, _ = estimate_local_lip_v2(model.model, trn_ds, top_norm=1, btm_norm=norm,
epsilon=auto_var.get_var("eps"), device=device)
result['avg_trn_lip_1'] = trn_lip
with Stopwatch("Estimating tst Lip"):
tst_lip, _ = estimate_local_lip_v2(model.model, tst_ds, top_norm=1, btm_norm=norm,
epsilon=auto_var.get_var("eps"), device=device)
result['avg_tst_lip_1'] = tst_lip
print(f"avg trn lip: {result['avg_trn_lip_1']}")
print(f"avg tst lip: {result['avg_tst_lip_1']}")
print(result)
return result
def exception_example():
print("[exception_example]")
try:
with Stopwatch():
raise ValueError("example error")
except ValueError:
pass
def cumulative_elapsed_time_example():
print(
"[cumulative_elapsed_time_example] use python logging module with different log level"
)
timer = Stopwatch("Waiting")
with timer:
sleep(1)
sleep(1)
with timer:
sleep(1)
timer.log_elapsed_time(
prefix="timer.log_elapsed_time(): ") # 0:00:01....
print("timer.get_elapsed_time():",
timer.get_elapsed_time()) # 0:00:01....
print("timer.split_elapsed_time:", timer.split_elapsed_time)
# [datetime.timedelta(seconds=1), datetime.timedelta(seconds=1)]
print("timer.get_cumulative_elapsed_time():",
timer.get_cumulative_elapsed_time())
def logging_example():
print("[logging_example] use python logging module")
with Stopwatch("Waiting", logger=logger):
sleep(1)
def changing_prefix_example():
print("[changing_prefix_example] change the prefix")
with Stopwatch("Waiting", prefix="[bistiming] "):
sleep(1)
def same_line_log_example():
print("[same_line_log_example] write the ending log at the same line")
with Stopwatch("Waiting", end_in_new_line=False):
sleep(1)
def hide_all_logs_example():
print("[hide_all_logs_example] hide all logs")
with Stopwatch(verbose=False):
sleep(1)
def hide_ending_log_example():
print("[hide_ending_log_example] hide ending log")
with Stopwatch("Waiting", verbose_end=False):
sleep(1)
def description_example():
print("[description_example] stopwatch with description")
with Stopwatch("Waiting"):
sleep(1)
def basic_context_manager_example():
print("[basic_context_manager_example]")
with Stopwatch():
sleep(1)
def basic_example():
print("[basic_example]")
timer = Stopwatch()
sleep(0.1)
timer.log_elapsed_time() # 0:00:00
timer.start()
sleep(0.1)
timer.log_elapsed_time() # 0:00:00.1
sleep(0.1)
timer.pause()
timer.log_elapsed_time() # 0:00:00.2
sleep(0.1)
timer.log_elapsed_time() # 0:00:00.2
timer.split() # 0:00:00.2
timer.log_elapsed_time() # 0:00:00
print("timer.get_cumulative_elapsed_time():",
timer.get_cumulative_elapsed_time()) # 0:00:00.2
sleep(0.1)
timer.start()
sleep(0.1)
timer.log_elapsed_time() # 0:00:00.1
print("timer.get_cumulative_elapsed_time():",
timer.get_cumulative_elapsed_time()) # 0:00:00.3
timer.split() # 0:00:00.1
sleep(0.1)
timer.pause()
timer.split() # 0:00:00.1
print("timer.get_cumulative_elapsed_time():",
timer.get_cumulative_elapsed_time()) # 0:00:00.4
print("timer.split_elapsed_time:",
[str(delta) for delta in timer.split_elapsed_time])
# [0:00:00.2, 0:00:00.1, 0:00:00.1]
timer.reset()
timer.log_elapsed_time() # 0:00:00
print("timer.get_cumulative_elapsed_time():",
timer.get_cumulative_elapsed_time()) # 0:00:00
print("timer.split_elapsed_time:", timer.split_elapsed_time) # []
sleep(0.1)
timer.start()
sleep(0.1)
timer.log_elapsed_time() # 0:00:00.1
def logging_level_example():
print(
"[logging_level_example] use python logging module with different log level"
)
with Stopwatch("Waiting", logger=logger, logging_level=logging.DEBUG):
sleep(1)
import numpy as np
import pickle
from bistiming import Stopwatch
direct_encoding_fields = ['hour', 'C1', 'C15', 'C16', 'C18', 'C20',
'banner_pos', 'site_category','app_category',
'device_type','device_conn_type']
frequency_encoding_fields = ['C14','C17', 'C19', 'C21',
'site_id','site_domain','app_id','app_domain',
'device_model', 'device_id']
with Stopwatch('Step 2: loading data')
# load direct encoding fields
with open('sets/click.pkl','rb') as f:
click = pickle.load(f)
with open('sets/hour.pkl','rb') as f:
hour = pickle.load(f)
with open('sets/C1.pkl','rb') as f:
C1 = pickle.load(f)
with open('sets/C15.pkl','rb') as f:
C15 = pickle.load(f)
with open('sets/C16.pkl','rb') as f:
C16 = pickle.load(f)
with open('sets/C18.pkl','rb') as f:
C18 = pickle.load(f)
def run_hypo(auto_var):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
_ = set_random_seed(auto_var)
norm = auto_var.get_var("norm")
trnX, trny, tstX, tsty = auto_var.get_var("dataset")
lbl_enc = OneHotEncoder(categories=[np.sort(np.unique(trny))],
sparse=False).fit(trny.reshape(-1, 1))
auto_var.set_intermidiate_variable("lbl_enc", lbl_enc)
n_classes = len(np.unique(trny))
n_channels = trnX.shape[-1]
result = {}
multigpu = False
model = auto_var.get_var("model",
trnX=trnX,
trny=trny,
multigpu=multigpu,
n_channels=n_channels)
model.tst_ds = (tstX, tsty)
with Stopwatch("Fitting Model"):
history = model.fit(trnX, trny)
result['trn_acc'] = (model.predict(trnX) == trny).mean()
result['tst_acc'] = (model.predict(tstX) == tsty).mean()
print(f"train acc: {result['trn_acc']}")
print(f"test acc: {result['tst_acc']}")
attack_model = auto_var.get_var(
"attack",
model=model,
n_classes=n_classes,
clip_min=0,
clip_max=1,
)
with Stopwatch("Attacking"):
adv_trnX = attack_model.perturb(trnX, trny)
adv_tstX = attack_model.perturb(tstX, tsty)
result['adv_trn_acc'] = (model.predict(adv_trnX) == trny).mean()
result['adv_tst_acc'] = (model.predict(adv_tstX) == tsty).mean()
print(f"adv trn acc: {result['adv_trn_acc']}")
print(f"adv tst acc: {result['adv_tst_acc']}")
del attack_model
with Stopwatch("Estimating trn Lip"):
trn_lip, _ = estimate_local_lip_v2(model.model,
trnX,
top_norm=1,
btm_norm=norm,
epsilon=auto_var.get_var("eps"),
device=device)
result['avg_trn_lip_1'] = trn_lip
with Stopwatch("Estimating tst Lip"):
tst_lip, _ = estimate_local_lip_v2(model.model,
tstX,
top_norm=1,
btm_norm=norm,
epsilon=auto_var.get_var("eps"),
device=device)
result['avg_tst_lip_1'] = tst_lip
print(f"avg trn lip: {result['avg_trn_lip_1']}")
print(f"avg tst lip: {result['avg_tst_lip_1']}")
print(result)
return result
def test_low_level_api():
timer = Stopwatch()
sleep(0.1)
assert timer.get_elapsed_time() == datetime.timedelta()
timer.start()
sleep(0.1)
assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.1)
sleep(0.1)
timer.pause()
assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.2)
sleep(0.1)
assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.2)
timer.split() # 0:00:00.2
assert timer.get_elapsed_time() == datetime.timedelta()
assert_timedelta_close_seconds(timer.get_cumulative_elapsed_time(), 0.2)
sleep(0.1)
timer.start()
sleep(0.1)
assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.1)
assert_timedelta_close_seconds(timer.get_cumulative_elapsed_time(), 0.3)
timer.split() # 0:00:00.1
sleep(0.1)
timer.pause()
timer.split() # 0:00:00.1
assert_timedelta_close_seconds(timer.get_cumulative_elapsed_time(), 0.4)
assert_timedelta_close_seconds(timer.split_elapsed_time[0], 0.2)
assert_timedelta_close_seconds(timer.split_elapsed_time[1], 0.1)
assert_timedelta_close_seconds(timer.split_elapsed_time[2], 0.1)
timer.reset()
assert timer.get_elapsed_time() == datetime.timedelta()
assert timer.get_cumulative_elapsed_time() == datetime.timedelta()
assert timer.split_elapsed_time == []
sleep(0.1)
timer.start()
sleep(0.1)
assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.1)