def run_restrictedImgnetHypo(auto_var): device = 'cuda' if torch.cuda.is_available() else 'cpu' _ = set_random_seed(auto_var) norm = auto_var.get_var("norm") trn_ds, tst_ds = auto_var.get_var("dataset") lbl_enc = None # TODO auto_var.set_intermidiate_variable("lbl_enc", lbl_enc) n_classes = len(trn_ds.classes) result = {} mock_trnX = np.concatenate([trn_ds[0][0], trn_ds[1][0]], axis=0) trny = np.array(trn_ds.targets) tsty = np.array(tst_ds.targets) #multigpu = True if torch.cuda.device_count() > 1 else False multigpu = False model = auto_var.get_var("model", trnX=trnX, trny=trny, multigpu=multigpu, n_channels=n_channels) model.tst_ds = (tstX, tsty) with Stopwatch("Fitting Model"): history = model.fit(trnX, trny) result['trn_acc'] = (model.predict_ds(trn_ds) == trny).mean() result['tst_acc'] = (model.predict_ds(tst_ds) == tsty).mean() print(f"train acc: {result['trn_acc']}") print(f"test acc: {result['tst_acc']}") attack_model = auto_var.get_var("attack", model=model, n_classes=n_classes, clip_min=0, clip_max=1,) with Stopwatch("Attacking Train"): adv_trnX = attack_model.perturb_ds(trn_ds) with Stopwatch("Attacking Test"): adv_tstX = attack_model.perturb_ds(tst_ds) result['adv_trn_acc'] = (model.predict(adv_trnX) == trny).mean() result['adv_tst_acc'] = (model.predict(adv_tstX) == tsty).mean() print(f"adv trn acc: {result['adv_trn_acc']}") print(f"adv tst acc: {result['adv_tst_acc']}") del attack_model with Stopwatch("Estimating trn Lip"): trn_lip, _ = estimate_local_lip_v2(model.model, trn_ds, top_norm=1, btm_norm=norm, epsilon=auto_var.get_var("eps"), device=device) result['avg_trn_lip_1'] = trn_lip with Stopwatch("Estimating tst Lip"): tst_lip, _ = estimate_local_lip_v2(model.model, tst_ds, top_norm=1, btm_norm=norm, epsilon=auto_var.get_var("eps"), device=device) result['avg_tst_lip_1'] = tst_lip print(f"avg trn lip: {result['avg_trn_lip_1']}") print(f"avg tst lip: {result['avg_tst_lip_1']}") print(result) return result
def exception_example(): print("[exception_example]") try: with Stopwatch(): raise ValueError("example error") except ValueError: pass
def cumulative_elapsed_time_example(): print( "[cumulative_elapsed_time_example] use python logging module with different log level" ) timer = Stopwatch("Waiting") with timer: sleep(1) sleep(1) with timer: sleep(1) timer.log_elapsed_time( prefix="timer.log_elapsed_time(): ") # 0:00:01.... print("timer.get_elapsed_time():", timer.get_elapsed_time()) # 0:00:01.... print("timer.split_elapsed_time:", timer.split_elapsed_time) # [datetime.timedelta(seconds=1), datetime.timedelta(seconds=1)] print("timer.get_cumulative_elapsed_time():", timer.get_cumulative_elapsed_time())
def basic_example(): print("[basic_example]") timer = Stopwatch() sleep(0.1) timer.log_elapsed_time() # 0:00:00 timer.start() sleep(0.1) timer.log_elapsed_time() # 0:00:00.1 sleep(0.1) timer.pause() timer.log_elapsed_time() # 0:00:00.2 sleep(0.1) timer.log_elapsed_time() # 0:00:00.2 timer.split() # 0:00:00.2 timer.log_elapsed_time() # 0:00:00 print("timer.get_cumulative_elapsed_time():", timer.get_cumulative_elapsed_time()) # 0:00:00.2 sleep(0.1) timer.start() sleep(0.1) timer.log_elapsed_time() # 0:00:00.1 print("timer.get_cumulative_elapsed_time():", timer.get_cumulative_elapsed_time()) # 0:00:00.3 timer.split() # 0:00:00.1 sleep(0.1) timer.pause() timer.split() # 0:00:00.1 print("timer.get_cumulative_elapsed_time():", timer.get_cumulative_elapsed_time()) # 0:00:00.4 print("timer.split_elapsed_time:", [str(delta) for delta in timer.split_elapsed_time]) # [0:00:00.2, 0:00:00.1, 0:00:00.1] timer.reset() timer.log_elapsed_time() # 0:00:00 print("timer.get_cumulative_elapsed_time():", timer.get_cumulative_elapsed_time()) # 0:00:00 print("timer.split_elapsed_time:", timer.split_elapsed_time) # [] sleep(0.1) timer.start() sleep(0.1) timer.log_elapsed_time() # 0:00:00.1
def test_low_level_api(): timer = Stopwatch() sleep(0.1) assert timer.get_elapsed_time() == datetime.timedelta() timer.start() sleep(0.1) assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.1) sleep(0.1) timer.pause() assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.2) sleep(0.1) assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.2) timer.split() # 0:00:00.2 assert timer.get_elapsed_time() == datetime.timedelta() assert_timedelta_close_seconds(timer.get_cumulative_elapsed_time(), 0.2) sleep(0.1) timer.start() sleep(0.1) assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.1) assert_timedelta_close_seconds(timer.get_cumulative_elapsed_time(), 0.3) timer.split() # 0:00:00.1 sleep(0.1) timer.pause() timer.split() # 0:00:00.1 assert_timedelta_close_seconds(timer.get_cumulative_elapsed_time(), 0.4) assert_timedelta_close_seconds(timer.split_elapsed_time[0], 0.2) assert_timedelta_close_seconds(timer.split_elapsed_time[1], 0.1) assert_timedelta_close_seconds(timer.split_elapsed_time[2], 0.1) timer.reset() assert timer.get_elapsed_time() == datetime.timedelta() assert timer.get_cumulative_elapsed_time() == datetime.timedelta() assert timer.split_elapsed_time == [] sleep(0.1) timer.start() sleep(0.1) assert_timedelta_close_seconds(timer.get_elapsed_time(), 0.1)
def logging_example(): print("[logging_example] use python logging module") with Stopwatch("Waiting", logger=logger): sleep(1)
def changing_prefix_example(): print("[changing_prefix_example] change the prefix") with Stopwatch("Waiting", prefix="[bistiming] "): sleep(1)
def same_line_log_example(): print("[same_line_log_example] write the ending log at the same line") with Stopwatch("Waiting", end_in_new_line=False): sleep(1)
def hide_all_logs_example(): print("[hide_all_logs_example] hide all logs") with Stopwatch(verbose=False): sleep(1)
def hide_ending_log_example(): print("[hide_ending_log_example] hide ending log") with Stopwatch("Waiting", verbose_end=False): sleep(1)
def description_example(): print("[description_example] stopwatch with description") with Stopwatch("Waiting"): sleep(1)
def basic_context_manager_example(): print("[basic_context_manager_example]") with Stopwatch(): sleep(1)
def logging_level_example(): print( "[logging_level_example] use python logging module with different log level" ) with Stopwatch("Waiting", logger=logger, logging_level=logging.DEBUG): sleep(1)
import numpy as np import pickle from bistiming import Stopwatch direct_encoding_fields = ['hour', 'C1', 'C15', 'C16', 'C18', 'C20', 'banner_pos', 'site_category','app_category', 'device_type','device_conn_type'] frequency_encoding_fields = ['C14','C17', 'C19', 'C21', 'site_id','site_domain','app_id','app_domain', 'device_model', 'device_id'] with Stopwatch('Step 2: loading data') # load direct encoding fields with open('sets/click.pkl','rb') as f: click = pickle.load(f) with open('sets/hour.pkl','rb') as f: hour = pickle.load(f) with open('sets/C1.pkl','rb') as f: C1 = pickle.load(f) with open('sets/C15.pkl','rb') as f: C15 = pickle.load(f) with open('sets/C16.pkl','rb') as f: C16 = pickle.load(f) with open('sets/C18.pkl','rb') as f: C18 = pickle.load(f)
def run_hypo(auto_var): device = 'cuda' if torch.cuda.is_available() else 'cpu' _ = set_random_seed(auto_var) norm = auto_var.get_var("norm") trnX, trny, tstX, tsty = auto_var.get_var("dataset") lbl_enc = OneHotEncoder(categories=[np.sort(np.unique(trny))], sparse=False).fit(trny.reshape(-1, 1)) auto_var.set_intermidiate_variable("lbl_enc", lbl_enc) n_classes = len(np.unique(trny)) n_channels = trnX.shape[-1] result = {} multigpu = False model = auto_var.get_var("model", trnX=trnX, trny=trny, multigpu=multigpu, n_channels=n_channels) model.tst_ds = (tstX, tsty) with Stopwatch("Fitting Model"): history = model.fit(trnX, trny) result['trn_acc'] = (model.predict(trnX) == trny).mean() result['tst_acc'] = (model.predict(tstX) == tsty).mean() print(f"train acc: {result['trn_acc']}") print(f"test acc: {result['tst_acc']}") attack_model = auto_var.get_var( "attack", model=model, n_classes=n_classes, clip_min=0, clip_max=1, ) with Stopwatch("Attacking"): adv_trnX = attack_model.perturb(trnX, trny) adv_tstX = attack_model.perturb(tstX, tsty) result['adv_trn_acc'] = (model.predict(adv_trnX) == trny).mean() result['adv_tst_acc'] = (model.predict(adv_tstX) == tsty).mean() print(f"adv trn acc: {result['adv_trn_acc']}") print(f"adv tst acc: {result['adv_tst_acc']}") del attack_model with Stopwatch("Estimating trn Lip"): trn_lip, _ = estimate_local_lip_v2(model.model, trnX, top_norm=1, btm_norm=norm, epsilon=auto_var.get_var("eps"), device=device) result['avg_trn_lip_1'] = trn_lip with Stopwatch("Estimating tst Lip"): tst_lip, _ = estimate_local_lip_v2(model.model, tstX, top_norm=1, btm_norm=norm, epsilon=auto_var.get_var("eps"), device=device) result['avg_tst_lip_1'] = tst_lip print(f"avg trn lip: {result['avg_trn_lip_1']}") print(f"avg tst lip: {result['avg_tst_lip_1']}") print(result) return result