def print_exit(success, attempts, t0): if not success: print(u.error(), "FAILED TO FIND CREDENTIALS") t = time.time() - t0 hours, rem = divmod(t, 3600) minutes, seconds = divmod(rem, 60) print("%s TIME: %dh%dm%ds" % (u.info(), hours, minutes, seconds)) print(u.info(), round(attempts / t, 2), "pw/s")
def parse_command_line_arguments(): argv = sys.argv[1:] try: opts, rest = getopt.getopt(argv, "", ["save=", "tag=", "show="]) except getopt.GetoptError as err: utils.error( "parse_command_line_arguments()", "The following error was thrown whilst parsing command-line arguments" ) utils.fatal("parse_command_line_arguments()", err) if len(rest) is not 1: raise ValueError( f"parse_command_line_arguments(): expected 1 unlabelled argument where {len(argv)} provided" ).with_traceback(sys.exc_info()[2]) cfg_name = rest[0] save_fname, do_show, tag = None, True, None if not utils.is_file(cfg_name): raise RuntimeError( f"parse_command_line_arguments(): config file {cfg_name} not found" ).with_traceback(sys.exc_info()[2]) for option, value in opts: if option in ["--tag"]: tag = str(value) utils.info("parse_command_line_arguments()", f"Labelling temporary files using the tag: {tag}") if option in ["--save"]: save_fname = str(value) utils.info("parse_command_line_arguments()", f"Opening plots file {save_fname}") utils.open_plots_pdf(save_fname) if option in ["--show"]: do_show = utils.string_to_object(value) if type(do_show) != bool: raise ValueError( f"parse_command_line_arguments(): --show value \"{value}\" could not be cast to a bool" ) return cfg_name, do_show, save_fname, tag
def brute_force(): attempts = 0 username = "******" passwords = list(read_passwords()) print(u.info(), 'TOT # PASSWORDS:', len(passwords)) session = build_session() try: for password in passwords: data = {USERNAME_FORM: username, PASSWORD_FORM: password} attempt = request("POST", session, data) attempts += 1 print_progress(attempts, username, password) if attempt.headers["Location"] == SUCCESS: print_success(username, password) return True, attempts except KeyboardInterrupt: print() pass return False, attempts
modals_data = pd.DataFrame( modal, columns=['Modal', 'cross_validation', 'n_jobs', 'scoring']) # %% print(modals_data) # %% df_holiday_package = utils.read_file(sheet_name) display(df_holiday_package) # %% df_holiday_package = utils.drop_columns_by_colnums(df_holiday_package, [0]) display(df_holiday_package) # %% utils.info(df_holiday_package) # %% categorical_columns = utils.get_categorical_cols(df_holiday_package) print(categorical_columns) # %% continuos_columns = utils.get_continous_cols(df_holiday_package) print(continuos_columns) # %% utils.convert_cat_into_code(df_holiday_package, categorical_columns) display(df_holiday_package) # %% utils.info(df_holiday_package)
# %% categorical_cols = utils.get_categorical_cols(df_gems) display(categorical_cols) # %% continous_cols = utils.get_continous_cols(df_gems) display(continous_cols) # %% utils.describe(df_gems, continous_cols) # %% utils.describe(df_gems, categorical_cols) # %% utils.info(df_gems) # %% utils.shape(df_gems) # %% utils.check_null_values(df_gems) # %% utils.replace_null_values_with_mean(df_gems) display(df_gems) # %% utils.check_null_values(df_gems) # %%
def do_general_setup(prog_name="do_general_setup()"): global num_scan_params, meas_dist, SM_model_dist, SM_exp_dist, SM_pred_dist, BSM_input_dists, BSM_scan_dists, param_grid # # parse arguments # utils.info(prog_name, "Parsing arguments") cfg_name, do_show_figs, plots_fname, tag = parse_command_line_arguments() # # read config file # utils.info(prog_name, f"Loading config file {cfg_name}") config.read_config(cfg_name) # # load scan param details # utils.info(prog_name, "Interpreting scan parameters") params = inputs.load_cfg_to_scan_params() num_scan_params = len(params) utils.info( prog_name, "Scan params are: " + " / ".join([p for p in config.get_scan_param_names()])) stats.set_target_coverage( config.config.getfloat("GENERAL", "ConfidenceLevel", fallback=0.95)) utils.info(prog_name, f"Target coverage is {stats.target_coverage}") # # load input files # utils.info(prog_name, "Loading input files") inputs.load_cfg_to_input_store(look_for_params=True) # # get input measured / expected / SM / BSM distributions # meas_dist = inputs.get_meas_dist(name="get_limits.py::meas") SM_model_dist = inputs.get_SM_dist(name="get_limits.py::SM::model", key="theoretical") SM_exp_dist = inputs.get_SM_dist(name="get_limits.py::SM::pred", key="experimental") SM_pred_dist = Distribution(SM_exp_dist) SM_pred_dist.cov = SM_pred_dist.cov + SM_model_dist.cov BSM_input_dists = inputs.get_BSM_distributions( prefix="get_limits.py::BSM::") utils.info(prog_name, f"Measured distribution is {meas_dist}") utils.info(prog_name, f"SM experimental distribution is {SM_exp_dist}") utils.info(prog_name, f"SM theoretical distribution is {SM_model_dist}") utils.info(prog_name, f"SM combined distribution is {SM_pred_dist}") for key, item in BSM_input_dists.items(): utils.info(prog_name, f"BSM input distribution at point {key} is {item}") # # generate model distributions across BSM grid # utils.info(prog_name, "Generating param grid") param_grid = create_param_grid(params) utils.info(prog_name, "Populating predictions across param grid") BSM_scan_dists = inputs.generate_BSM_predictions(BSM_input_dists, param_grid, SM=SM_model_dist) utils.info(prog_name, "Adding SM to BSM") shape = BSM_scan_dists.values.shape BSM_scan_dists.values = BSM_scan_dists.values.flatten() for idx in range(len(BSM_scan_dists.values)): BSM_scan_dists.values[idx] = BSM_scan_dists.values[ idx] + SM_model_dist # not using add_to_values as I want SM error to be included utils.info( prog_name, f"BSM scan distribution at index {idx} is {BSM_scan_dists.values[idx]}" ) BSM_scan_dists.values = BSM_scan_dists.values.reshape(shape) # # return config options # return cfg_name, do_show_figs, plots_fname, tag
def main(): # # config and setup # cfg_name, do_show_figs, plots_fname, tag = do_general_setup( prog_name="get_limits.py") global num_scan_params, meas_dist, SM_model_dist, SM_exp_dist, BSM_input_dists, BSM_scan_dists, param_grid # # get SM expected limit # utils.info("get_limits.py", "Getting expected confidence levels for scan points") grid_of_pred_CL = stats.get_CL_across_grid(SM_exp_dist, BSM_scan_dists, SM_dist=SM_model_dist) # # get observed limit # utils.info("get_limits.py", "Getting observed confidence levels for scan points") grid_of_obs_CL = stats.get_CL_across_grid(meas_dist, BSM_scan_dists, SM_dist=SM_model_dist) if num_scan_params == 1: utils.info( "get_limits.py", f"Observed {100.*stats.target_coverage:.2f}% confidence limit is {stats.get_limit_from_levels(grid_of_obs_CL )}" ) utils.info( "get_limits.py", f"Expected {100.*stats.target_coverage:.2f}% confidence limit is {stats.get_limit_from_levels(grid_of_pred_CL)}" ) # # generate SM toys and get limits # utils.info("get_limits()", "Getting expected spread of confidence levels for scan points") n_toys = config.get_and_enforce("STEERING", "SM.experimental.ntoys", to_type=int) if n_toys < 2: raise ValueError( f"get_limits(): SM.experimental.ntoys must be at least 2 ({n_toys} provided)" ) utils.info( "get_limits()", f"Throwing {n_toys} toys around the experimental SM expectation") SM_toy_limits, SM_coverage_grid = get_toy_spread_of_limits(SM_exp_dist, SM_model_dist, BSM_scan_dists, n_toys, tag=tag) if num_scan_params == 1: utils.info( "get_limits.py", f"Median {100.*stats.target_coverage:.2f}% limit of SM toys is {SM_toy_limits[int(0.5*n_toys)]:.0f}" ) utils.info( "get_limits.py", f"16th percentile {100.*stats.target_coverage:.2f}% limit of SM toys is {SM_toy_limits[int(0.16*n_toys)]:.0f}" ) utils.info( "get_limits.py", f"84th percentile {100.*stats.target_coverage:.2f}% limit of SM toys is {SM_toy_limits[int(0.84*n_toys)]:.0f}" ) # # plot # utils.set_mpl_style() fig = plt.figure() ax = fig.add_subplot(1, 1, 1) if num_scan_params == 1: limit_obs = stats.get_limit_from_levels(grid_of_obs_CL) limit_SM = stats.get_limit_from_levels(grid_of_pred_CL) limit_toys_5pc, limit_toys_16pc, limit_toys_median, limit_toys_84pc, limit_toys_95pc = SM_toy_limits[ int(0.05 * n_toys)], SM_toy_limits[int( 0.16 * n_toys)], SM_toy_limits[int( 0.5 * n_toys)], SM_toy_limits[int( 0.84 * n_toys)], SM_toy_limits[int(0.95 * n_toys)] plt.axvspan(limit_toys_5pc, limit_toys_95pc, color="darkorange", linestyle=None) plt.axvspan(limit_toys_16pc, limit_toys_84pc, color="gold", linestyle=None) plt.plot([limit_toys_median, limit_toys_median], [0, 1], color="darkblue", linestyle="dashed", linewidth=1) plt.plot([limit_SM, limit_SM], [0, 1], color="green") plt.plot([limit_obs, limit_obs], [0, 1], color="purple") ax.yaxis.set_visible(False) ax.set_ylim([0, 1]) else: plt.contourf(SM_coverage_grid.axes[0], SM_coverage_grid.axes[1], SM_coverage_grid.values.transpose(), [0.05, 0.16, 0.84, 0.95], linestyles=None, colors=["gold", "darkorange", "gold"]) plt.contour(SM_coverage_grid.axes[0], SM_coverage_grid.axes[1], SM_coverage_grid.values.transpose(), [0.5], linestyles="dashed", colors=["darkblue"], linewidths=1) plt.contour(grid_of_pred_CL.axes[0], grid_of_pred_CL.axes[1], grid_of_pred_CL.values.transpose(), [1.0 - stats.target_coverage], colors="green") plt.contour(grid_of_obs_CL.axes[0], grid_of_obs_CL.axes[1], grid_of_obs_CL.values.transpose(), [1.0 - stats.target_coverage], colors="purple") '''for l in SM_toy_limits : l = l[0] plt.plot(l[:,0], l[:,1], "-", color="gold", linewidth=0.1, alpha=1)''' ax.set_ylabel( f"{inputs.scan_params[1].label} [{inputs.scan_params[1].units}]") plt.ylabel( f"{inputs.scan_params[1].label} [{inputs.scan_params[1].units}]", horizontalalignment='right', y=1.0, fontsize="large") format_axis_from_config("GET_LIMITS") ax.set_xlabel( f"{inputs.scan_params[0].label} [{inputs.scan_params[0].units}]") plt.xlabel( f"{inputs.scan_params[0].label} [{inputs.scan_params[0].units}]", horizontalalignment='right', x=1.0, fontsize="large") plt.legend([ Line2D([0], [0], color="purple", lw=2), Line2D([0], [0], color="green", lw=2), Line2D([0], [0], color="darkblue", linestyle="dashed", lw=1), Patch(color="gold", linestyle=None), Patch(color="darkorange", linestyle=None) ], [ f"Obs. ({100*stats.target_coverage:.0f}% $CL_s$)", f"Exp. ({100*stats.target_coverage:.0f}% $CL_s$)", "SM toys: median", "SM toys: 68% coverage", "SM toys: 95% coverage" ], loc=utils.string_to_object( config.config.get("GET_LIMITS", "legend.position", fallback="\'best\'"))) if do_show_figs: plt.show() if plots_fname is not None: utils.save_figure(fig) utils.close_plots_pdf()
def get_toy_spread_of_limits(SM_exp_dist, SM_model_dist, BSM_scan_dists, n_toys, target_coverage=None, tag=None): global param_grid do_store_toys = config.config.getboolean("GENERAL", "SaveAndLoadToys", fallback=False) if tag is None: fname = ".get_limits__get_toy_spread_of_1D_limits.pickle" else: fname = f".get_limits__get_toy_spread_of_1D_limits.{tag}.pickle" if target_coverage is None: target_coverage = stats.get_target_coverage() required_kwargs = {} required_kwargs["test_stat"] = config.get_and_enforce( "GENERAL", "TestStatistic", from_selection=stats.test_statistics) required_kwargs["limits_method"] = config.get_and_enforce( "GENERAL", "LimitsMethod", from_selection=stats.limit_methods) required_kwargs["test_stat_strategy"] = config.get_and_enforce( "GENERAL", "TestStatistic.Distribution", from_selection=stats.test_stat_strategies) required_kwargs["target_coverage"] = target_coverage required_kwargs["n_toys"] = n_toys required_kwargs["n_axes"] = len(BSM_scan_dists.axes) for idx in range(len(BSM_scan_dists.keys)): required_kwargs[f"axis.index_{idx}"] = BSM_scan_dists.keys[idx] for axis, key in zip(BSM_scan_dists.axes, BSM_scan_dists.keys): required_kwargs[f"{key}.length"] = len(axis) for idx in range(len(axis)): required_kwargs[f"{key}.index_{idx}"] = axis[idx] if do_store_toys: load_success, pickle_file = utils.open_from_pickle( fname, **required_kwargs) if load_success: utils.info("get_toy_spread_of_1D_limits()", f"Sucessfully loaded toy limits from {fname}") return pickle_file["limits"], pickle_file["coverage"] SM_exp_toys = SM_pred_dist.generate_toys(n_toys) utils.info("get_toy_spread_of_1D_limits()", "Getting expected confidence levels for toys") SM_toy_limits = [] grid_of_coverage = Grid(param_grid) grid_of_coverage.generate(dtype=np.float64) tmp_array_of_coverage = grid_of_coverage.values.flatten() for toy_idx in range(n_toys): grid_of_this_CL = stats.get_CL_across_grid(SM_exp_toys[toy_idx], BSM_scan_dists, SM_dist=SM_model_dist) limit = stats.get_limit_from_levels(grid_of_this_CL, target_coverage) SM_toy_limits.append(limit) flattened_grid_of_this_CL = grid_of_this_CL.values.flatten() for i in range(len(tmp_array_of_coverage)): this_CL = flattened_grid_of_this_CL[i] if this_CL > (1.0 - target_coverage): continue tmp_array_of_coverage[i] = tmp_array_of_coverage[i] + 1 pct_complete = 100. * (1 + toy_idx) / n_toys if pct_complete % 10 == 0: utils.info("get_toy_spread_of_1D_limits()", f"{int(pct_complete)}% toys processed") tmp_array_of_coverage = tmp_array_of_coverage / float(n_toys) grid_of_coverage.values = tmp_array_of_coverage.reshape( grid_of_coverage.values.shape) if len(BSM_scan_dists.axes) == 1: SM_toy_limits.sort() if do_store_toys: utils.save_to_pickle(fname, { "limits": SM_toy_limits, "coverage": grid_of_coverage }, **required_kwargs) return SM_toy_limits, grid_of_coverage
def print_exit(attempts): t = time.time() - t0 hours, rem = divmod(t, 3600) minutes, seconds = divmod(rem, 60) print('%s TIME: %dh%dm%ds' % (u.info(), hours, minutes, seconds)) print(u.info(), round(attempts / t, 2), 'pw/s')
def print_success(attempts, username, password): print(u.info(), 'FOUND CREDENTIALS') print(u.info(), 'USERNAME:'******'PASSWORD:', password) print_exit(attempts)
def print_info(num_of_passwords, num_of_threads): print(u.info(), 'TOT # PASSWORDS:', num_of_passwords) print(u.info(), 'TOT # THREADS:', num_of_threads)
def print_tokens(csrf, session_cookie): print(u.info(), 'FOUND TOKENS') print(u.info(), CSRF_TOKEN, csrf) print(u.info(), SESSION_COOKIE, session_cookie)
def print_success(username, password): print() print(u.info(), "FOUD CREDENTIALS") print(u.info(), "USERNAME:"******"PASSWORD:", password)
def print_tokens(csrf, jsession_id): print(u.info(), "FOUND TOKENS") print(u.info(), "_csrf:", csrf) print(u.info(), "JSESSIONID:", jsession_id)