def pack_a_question_to_send(self): questions = self.questions data_to_send = dict() data_to_send["question"] = "" data_to_send["input"] = list() data_to_send["output"] = list() # Get random question self.current_question = questions[random.randint( 0, len(questions) - 1)] q_id = self.current_question[0] question = self.current_question[1] data_to_send["question"] = question data_to_send["input"] = list() data_to_send["output"] = list() # Get answers of the question testcases = self.get_testcases(q_id) for i in testcases: data_to_send["input"].append(i[1]) data_to_send["output"].append(i[2]) return utils.dict_to_str(data_to_send)
def train(df, attrs, clf_class, clf_name, model_params, mode, magic_number, dates, dataset_name, trading_params): trade_freq = trading_params['trade_frequency'] name = '%s-%s-attr%s-%s-%s-%s-%s-%s_' % ( clf_name, dataset_name, len(attrs), dict_to_str(model_params).replace( ' ', '_').replace(':', ''), mode, magic_number, pd.to_datetime(dates[0], format=DATE_FORMAT).date(), pd.to_datetime(dates[1], format=DATE_FORMAT).date()) cached_file = os.path.join(CACHE_PATH + '/models/', name) start_date, final_date = dates idx = 0 indices = sorted([ day for day in list(set(df.index.values)) if start_date <= day <= final_date ]) print("Model and params: %s %s " % (clf_name, model_params)) # magic number is by default 53, 52 weeks for training 1 for prediction while idx + magic_number < len(indices) and indices[idx + magic_number] <= \ indices[-1]: if mode == CLASSIFICATION: train_x, train_y, test_x, test_y = \ get_classification_data(clf_name, df, attrs, indices, idx, magic_number) elif mode == REGRESSION: # get regression datasets (target is float y -> ratio of increase) train_x, train_y, test_x, test_y = \ get_regression_data(clf_name, df, attrs, indices, idx, magic_number) print( "Training %s/%s with %s instances." % (idx // trade_freq, len(indices) // trade_freq, train_x.shape[0])) sys.stdout.flush() clf_cached_file = cached_file + str(indices[idx])[:10] if not CHECKPOINTING: clf = clf_class(**model_params).fit(train_x, train_y) else: try: clf = load_obj(clf_cached_file) except: clf = clf_class(**model_params).fit(train_x, train_y) save_obj(clf, clf_cached_file) pred = clf.predict(test_x) # import ipdb # ipdb.set_trace() df.loc[indices[idx + magic_number], clf_name] = pred idx += trade_freq df_trade = df.dropna(axis=0) print("Finished training for %s" % (clf_name)) return df_trade
def create_backup_plan(self, **kwargs): try: plan = BackupPlan() plan.created_date = date_now() plan.description = get_validate_arg(kwargs, "description", expected_type=(str, unicode), required=False) plan.strategy = get_validate_arg(kwargs, "strategy", expected_type=BackupStrategy) plan.schedule = get_validate_arg(kwargs, "schedule", expected_type=AbstractSchedule) plan.source = get_validate_arg(kwargs, "source", expected_type=BackupSource) plan.target = get_validate_arg(kwargs, "target", expected_type=BackupTarget) plan.retention_policy = get_validate_arg(kwargs, "retention_policy", expected_type= RetentionPolicy, required=False) plan.priority = get_validate_arg(kwargs, "priority", expected_type=(int, long, float, complex), required=False) plan.secondary_targets = get_validate_arg(kwargs, "secondary_targets", expected_type=list, required=False) # tags plan.tags = get_validate_arg(kwargs, "tags", expected_type=dict, required=False) plan_doc = plan.to_document() get_mbs().plan_collection.save_document(plan_doc) # set the backup plan id from the saved doc plan.id = plan_doc["_id"] self.info("Saved backup plan \n%s" % plan) # process plan to set next occurrence self._scheduler._process_plan(plan) return plan except Exception, e: args_str = dict_to_str(kwargs) msg = ("Failed to create plan. Args:\n %s" % args_str) logger.error(msg) logger.error(traceback.format_exc()) raise CreatePlanError(msg=msg, cause=e)
def log_info(args, classifiers, datasets): print(" * Arguments:") for k,v in args.__dict__.items(): print("\t- %s: %s" % (k.title(), v)) print("\n\n * Models to train: [%s]" % np.sum( [len(v[1]) for v in classifiers.values()])) for k,v in classifiers.items(): print('\n\t- ' + k) for p in [e for e in v[1] if e]: print('\t\t' + dict_to_str(p)) print("\n\n * Datasets created: %s: \n%s\n" % ( len(datasets), '\n\t- '.join(datasets.keys())))
def get_output_stats(self, data=None): """ returns final stats, as dict of strings """ if not data: data = self.analyze() requests = data['requests_count'] time_delta = data['last_datetime'] - data['first_datetime'] req_per_sec = str(round(requests / time_delta.seconds, 3)) twohoundreds_avg_size = humanize(data['2XX_total_size'] // data['2XX_count']) response_status = dict_to_str(data['response_status_count']) return { 'requests': str(requests), 'status_count': response_status, 'request_per_second': req_per_sec, '2XX_avg_size': twohoundreds_avg_size, }
def fit_and_predict(self, X_train, X_test, y_train, y_test): # BROOF model training & testing broof_cv = self.broof_classiefier() best_broof, best_params, train_time = self.randomized_search_fit( broof_cv, X_train, y_train) best_params_str = utils.dict_to_str(best_params) value_dict = self.predicting(best_broof, X_test, y_test) self.write_result_table_to_file('BROOF', best_params_str, value_dict, train_time) # XGBoost model training & testing xgb_cv = self.xgb_classifier() best_xgb, best_params, train_time = self.randomized_search_fit( xgb_cv, X_train, y_train) value_dict = self.predicting(best_xgb, X_test, y_test) self.write_result_table_to_file('XGBoost', best_params_str, value_dict, train_time)
def _ii(self, table: str, values: dict, conditions: dict = None): """ :type table: str :type values: dict :type conditions: dict """ cur = self.DB.cursor() if conditions is None: keys, values = dict_to_str(values) cur.execute(f"INSERT INTO {table} ({keys}) VALUES ({values})") else: # Update u_values = values.copy() for c in conditions: try: u_values.pop(c) except KeyError: pass # debug = f"UPDATE {table} SET {kcv(u_values, '=')} WHERE {kcv(conditions, '=', ' AND ')}" cur.execute( f"UPDATE {table} SET {kcv(u_values, '=')} WHERE {kcv(conditions, '=', ' AND ')}" )
def add_hash(remain_dict, word_branch): '''Get a unique representation of remain_dict and add reference WordBranch. Args remain_dict ({char => int}) The remaining letters of the phrase from this point in the tree. word_branch (WordBranch) The WordBranch we're working on. Returns (bool) Return True if the remain_dict was a new entry in the dictionary. ''' hash_to_branch = WordBranch.get_hash_to_branch() dict_str = utils.dict_to_str(remain_dict) if dict_str in hash_to_branch: wb = hash_to_branch[dict_str] if wb.valid_children != None and len(wb.valid_children) > 0: mark_path(word_branch) word_branch.valid_children = wb.valid_children return False else: hash_to_branch[dict_str] = word_branch return True
def ii(self, table, vals, cur, conditions=None): """ Inserts into the table vals using cur :param table: The table you're inserting into :param vals: The keys represent the titles and values the values inserted into said columns :param cur: The cursor used to communicate with the db :param conditions: The names of the columns you're checking in n,0 and their values in n,1 """ if conditions is None: keys, values = dict_to_str(vals) # debug = f"INSERT INTO {table} ({keys}) VALUES ({values})" cur.execute(f"INSERT INTO {table} ({keys}) VALUES ({values})") else: # Update u_vals = vals.copy() for c in conditions: try: u_vals.pop(c) except KeyError: pass # debug = f"UPDATE {table} SET {kcv(u_vals, '=')} WHERE {kcv(conditions, '=', ' AND ')}" cur.execute( f"UPDATE {table} SET {kcv(u_vals, '=')} WHERE {kcv(conditions, '=', ' AND ')}" )
self.set_custom_backup_props(backup) backup_doc = backup.to_document() get_mbs().backup_collection.save_document(backup_doc) # set the backup id from the saved doc backup.id = backup_doc["_id"] self.info("Saved backup \n%s" % backup) if backup.state == State.FAILED: trigger_task_finished_event(backup, State.FAILED) return backup except Exception, e: args_str = dict_to_str(kwargs) msg = ("Failed to schedule backup. Args:\n %s" % args_str) logger.error(msg) logger.error(traceback.format_exc()) raise BackupSchedulingError(msg=msg, cause=e) ########################################################################### def set_custom_backup_props(self, backup): pass ########################################################################### def force_expire_backup(self, backup): self.backup_expiration_manager.expire_backup(backup, force=True) ########################################################################### def cancel_backup(self, backup):
def test_dict_to_str(): dict1 = {'p':1, 't':4, 'a':0, 'i':2} dict_str = utils.dict_to_str(dict1) assert dict_str == "pttttii"
(128, [2400, 8000], params_128)], desc='ImgSizes', leave=True, disable=args.disable_tqdm): for n_measure in tqdm(n_measures, desc='N_measures', leave=False, disable=args.disable_tqdm): img_shape = (3, img_size, img_size) forward_model = GaussianCompressiveSensing(n_measure=n_measure, img_shape=img_shape) # forward_model = NoOp() for img_name in tqdm(os.listdir(args.img_dir), desc='Images', leave=False, disable=args.disable_tqdm): orig_img = load_target_image( os.path.join(args.img_dir, img_name), img_size).to(DEVICE) img_basename, _ = os.path.splitext(img_name) x_hat, x_degraded, _ = deep_decoder_recover( orig_img, forward_model, run_dir='deep_decoder', run_name=(img_basename + args.run_name_suffix + '.' + dict_to_str(params) + f'.n_measure={n_measure}.img_size={img_size}'), img_size=img_size, **params)
def deep_decoder_images(args): if args.set_seed: torch.manual_seed(0) np.random.seed(0) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False os.makedirs(BASE_DIR, exist_ok=True) metadata = recovery_settings[args.model] forwards = forward_models[args.model] data_split = Path(args.img_dir).name for img_name in tqdm(sorted(os.listdir(args.img_dir)), desc='Images', leave=True, disable=args.disable_tqdm): orig_img = load_target_image(os.path.join(args.img_dir, img_name), metadata['img_size']).to(DEVICE) img_basename, _ = os.path.splitext(img_name) for f, f_args_list in tqdm(forwards.items(), desc='Forwards', leave=False, disable=args.disable_tqdm): for f_args in tqdm(f_args_list, desc=f'{f} Args', leave=False, disable=args.disable_tqdm): f_args['img_shape'] = (3, metadata['img_size'], metadata['img_size']) forward_model = get_forward_model(f, **f_args) recovered_name = 'recovered.pt' results_folder = get_results_folder( image_name=img_basename, model=args.model, n_cuts=0, # NOTE - this field is unused for iagan split=data_split, forward_model=forward_model, recovery_params=dict_to_str(metadata), base_dir=BASE_DIR) os.makedirs(results_folder, exist_ok=True) recovered_path = results_folder / recovered_name if os.path.exists(recovered_path) and not args.overwrite: print(f'{recovered_path} already exists, skipping...') continue if args.run_name is not None: current_run_name = (f'{img_basename}' + f'.{forward_model}' + dict_to_str(metadata) + f'.{args.run_name}') else: current_run_name = None recovered_img, distorted_img, _ = deep_decoder_recover( orig_img, forward_model=forward_model, optimizer=metadata['optimizer'], num_filters=metadata['num_filters'], depth=metadata['depth'], lr=metadata['lr'], img_size=metadata['img_size'], steps=metadata['steps'], restarts=metadata['restarts'], run_dir=args.run_dir, run_name=current_run_name, disable_tqdm=args.disable_tqdm) # Make images folder img_folder = get_images_folder(split=data_split, image_name=img_basename, img_size=metadata['img_size'], base_dir=BASE_DIR) os.makedirs(img_folder, exist_ok=True) # Save original image if needed original_img_path = img_folder / 'original.pt' if not os.path.exists(original_img_path): torch.save(orig_img, original_img_path) # Save distorted image if needed if forward_model.viewable: distorted_img_path = img_folder / f'{forward_model}.pt' if not os.path.exists(distorted_img_path): torch.save(distorted_img, distorted_img_path) # Save recovered image and metadata torch.save(recovered_img, recovered_path) pickle.dump(metadata, open(results_folder / 'metadata.pkl', 'wb')) p = psnr(recovered_img, orig_img) pickle.dump(p, open(results_folder / 'psnr.pkl', 'wb'))
def mgan_images(args): if args.set_seed: torch.manual_seed(0) np.random.seed(0) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False os.makedirs(BASE_DIR, exist_ok=True) if args.model in ['mgan_began_cs']: gen = Generator128(64) gen = load_trained_net( gen, ('./checkpoints/celeba_began.withskips.bs32.cosine.min=0.25' '.n_cuts=0/gen_ckpt.49.pt')) gen = gen.eval().to(DEVICE) img_size = 128 elif args.model in ['mgan_vanilla_vae_cs']: gen = VAE() t = torch.load('./vae_checkpoints/vae_bs=128_beta=1.0/epoch_19.pt') gen.load_state_dict(t) gen = gen.eval().to(DEVICE) gen = gen.decoder img_size = 128 elif args.model in ['mgan_dcgan_cs']: gen = dcgan_generator() t = torch.load(('./dcgan_checkpoints/netG.epoch_24.n_cuts_0.bs_64' '.b1_0.5.lr_0.0002.pt')) gen.load_state_dict(t) gen = gen.eval().to(DEVICE) img_size = 64 else: raise NotImplementedError() img_shape = (3, img_size, img_size) metadata = recovery_settings[args.model] n_cuts_list = metadata['n_cuts_list'] del (metadata['n_cuts_list']) z_init_mode_list = metadata['z_init_mode'] limit_list = metadata['limit'] assert len(z_init_mode_list) == len(limit_list) del (metadata['z_init_mode']) del (metadata['limit']) forwards = forward_models[args.model] data_split = Path(args.img_dir).name for img_name in tqdm(sorted(os.listdir(args.img_dir)), desc='Images', leave=True, disable=args.disable_tqdm): # Load image and get filename without extension orig_img = load_target_image(os.path.join(args.img_dir, img_name), img_size).to(DEVICE) img_basename, _ = os.path.splitext(img_name) for n_cuts in tqdm(n_cuts_list, desc='N_cuts', leave=False, disable=args.disable_tqdm): metadata['n_cuts'] = n_cuts for i, (f, f_args_list) in enumerate( tqdm(forwards.items(), desc='Forwards', leave=False, disable=args.disable_tqdm)): for f_args in tqdm(f_args_list, desc=f'{f} Args', leave=False, disable=args.disable_tqdm): f_args['img_shape'] = img_shape forward_model = get_forward_model(f, **f_args) for z_init_mode, limit in zip( tqdm(z_init_mode_list, desc='z_init_mode', leave=False), limit_list): metadata['z_init_mode'] = z_init_mode metadata['limit'] = limit # Before doing recovery, check if results already exist # and possibly skip recovered_name = 'recovered.pt' results_folder = get_results_folder( image_name=img_basename, model=args.model, n_cuts=n_cuts, split=data_split, forward_model=forward_model, recovery_params=dict_to_str(metadata), base_dir=BASE_DIR) os.makedirs(results_folder, exist_ok=True) recovered_path = results_folder / recovered_name if os.path.exists( recovered_path) and not args.overwrite: print( f'{recovered_path} already exists, skipping...' ) continue if args.run_name is not None: current_run_name = ( f'{img_basename}.{forward_model}' f'.{dict_to_str(metadata)}' f'.{args.run_name}') else: current_run_name = None recovered_img, distorted_img, _ = mgan_recover( orig_img, gen, n_cuts, forward_model, metadata['optimizer'], z_init_mode, limit, metadata['z_lr'], metadata['n_steps'], metadata['z_number'], metadata['restarts'], args.run_dir, current_run_name, args.disable_tqdm) # Make images folder img_folder = get_images_folder(split=data_split, image_name=img_basename, img_size=img_size, base_dir=BASE_DIR) os.makedirs(img_folder, exist_ok=True) # Save original image if needed original_img_path = img_folder / 'original.pt' if not os.path.exists(original_img_path): torch.save(orig_img, original_img_path) # Save distorted image if needed if forward_model.viewable: distorted_img_path = img_folder / f'{forward_model}.pt' if not os.path.exists(distorted_img_path): torch.save(distorted_img, distorted_img_path) # Save recovered image and metadata torch.save(recovered_img, recovered_path) pickle.dump( metadata, open(results_folder / 'metadata.pkl', 'wb')) p = psnr(recovered_img, orig_img) pickle.dump(p, open(results_folder / 'psnr.pkl', 'wb'))