Exemplo n.º 1
0
    def pack_a_question_to_send(self):
        questions = self.questions
        data_to_send = dict()
        data_to_send["question"] = ""
        data_to_send["input"] = list()
        data_to_send["output"] = list()

        # Get random question
        self.current_question = questions[random.randint(
            0,
            len(questions) - 1)]

        q_id = self.current_question[0]
        question = self.current_question[1]
        data_to_send["question"] = question
        data_to_send["input"] = list()
        data_to_send["output"] = list()

        # Get answers of the question
        testcases = self.get_testcases(q_id)
        for i in testcases:
            data_to_send["input"].append(i[1])
            data_to_send["output"].append(i[2])

        return utils.dict_to_str(data_to_send)
Exemplo n.º 2
0
def train(df, attrs, clf_class, clf_name, model_params, mode, magic_number,
          dates, dataset_name, trading_params):
    trade_freq = trading_params['trade_frequency']
    name = '%s-%s-attr%s-%s-%s-%s-%s-%s_' % (
        clf_name, dataset_name, len(attrs), dict_to_str(model_params).replace(
            ' ', '_').replace(':', ''), mode, magic_number,
        pd.to_datetime(dates[0], format=DATE_FORMAT).date(),
        pd.to_datetime(dates[1], format=DATE_FORMAT).date())
    cached_file = os.path.join(CACHE_PATH + '/models/', name)

    start_date, final_date = dates
    idx = 0

    indices = sorted([
        day for day in list(set(df.index.values))
        if start_date <= day <= final_date
    ])

    print("Model and params: %s %s " % (clf_name, model_params))
    # magic number is by default 53, 52 weeks for training 1 for prediction
    while idx + magic_number < len(indices) and indices[idx + magic_number] <= \
            indices[-1]:

        if mode == CLASSIFICATION:
            train_x, train_y, test_x, test_y = \
                get_classification_data(clf_name, df, attrs, indices, idx,
                                        magic_number)
        elif mode == REGRESSION:
            # get regression datasets (target is float y -> ratio of increase)
            train_x, train_y, test_x, test_y = \
                get_regression_data(clf_name, df, attrs, indices, idx,
                                    magic_number)

        print(
            "Training %s/%s with %s instances." %
            (idx // trade_freq, len(indices) // trade_freq, train_x.shape[0]))
        sys.stdout.flush()

        clf_cached_file = cached_file + str(indices[idx])[:10]

        if not CHECKPOINTING:
            clf = clf_class(**model_params).fit(train_x, train_y)
        else:
            try:
                clf = load_obj(clf_cached_file)
            except:
                clf = clf_class(**model_params).fit(train_x, train_y)
                save_obj(clf, clf_cached_file)

        pred = clf.predict(test_x)

        # import ipdb
        # ipdb.set_trace()
        df.loc[indices[idx + magic_number], clf_name] = pred

        idx += trade_freq
    df_trade = df.dropna(axis=0)

    print("Finished training for %s" % (clf_name))
    return df_trade
    def create_backup_plan(self, **kwargs):
        try:
            plan = BackupPlan()
            plan.created_date = date_now()

            plan.description = get_validate_arg(kwargs, "description",
                                             expected_type=(str, unicode),
                                             required=False)

            plan.strategy = get_validate_arg(kwargs, "strategy",
                                             expected_type=BackupStrategy)


            plan.schedule = get_validate_arg(kwargs, "schedule",
                                             expected_type=AbstractSchedule)

            plan.source = get_validate_arg(kwargs, "source",
                                           expected_type=BackupSource)

            plan.target = get_validate_arg(kwargs, "target",
                                           expected_type=BackupTarget)

            plan.retention_policy = get_validate_arg(kwargs, "retention_policy",
                                                     expected_type=
                                                     RetentionPolicy,
                                                     required=False)

            plan.priority = get_validate_arg(kwargs, "priority",
                                             expected_type=(int, long,
                                                            float, complex),
                                             required=False)

            plan.secondary_targets = get_validate_arg(kwargs,
                                                      "secondary_targets",
                                                      expected_type=list,
                                                      required=False)

            # tags
            plan.tags = get_validate_arg(kwargs, "tags", expected_type=dict,
                                         required=False)

            plan_doc = plan.to_document()
            get_mbs().plan_collection.save_document(plan_doc)
            # set the backup plan id from the saved doc

            plan.id = plan_doc["_id"]

            self.info("Saved backup plan \n%s" % plan)
            # process plan to set next occurrence
            self._scheduler._process_plan(plan)
            return plan
        except Exception, e:
            args_str = dict_to_str(kwargs)
            msg = ("Failed to create plan. Args:\n %s" % args_str)
            logger.error(msg)
            logger.error(traceback.format_exc())
            raise CreatePlanError(msg=msg, cause=e)
Exemplo n.º 4
0
def log_info(args, classifiers, datasets):
    print(" * Arguments:")
    for k,v in args.__dict__.items():
        print("\t- %s: %s" % (k.title(), v))
    print("\n\n * Models to train: [%s]" % np.sum(
        [len(v[1]) for v in classifiers.values()]))
    for k,v in classifiers.items():
        print('\n\t- ' + k)
        for p in [e for e in v[1] if e]:
            print('\t\t' + dict_to_str(p))
    print("\n\n * Datasets created: %s: \n%s\n" % (
        len(datasets), '\n\t- '.join(datasets.keys())))
Exemplo n.º 5
0
 def get_output_stats(self, data=None):
     """ returns final stats, as dict of strings """
     if not data:
         data = self.analyze()
     requests = data['requests_count']
     time_delta = data['last_datetime'] - data['first_datetime']
     req_per_sec = str(round(requests / time_delta.seconds, 3))
     twohoundreds_avg_size = humanize(data['2XX_total_size'] //
                                      data['2XX_count'])
     response_status = dict_to_str(data['response_status_count'])
     return {
         'requests': str(requests),
         'status_count': response_status,
         'request_per_second': req_per_sec,
         '2XX_avg_size': twohoundreds_avg_size,
     }
Exemplo n.º 6
0
    def fit_and_predict(self, X_train, X_test, y_train, y_test):
        # BROOF model training & testing
        broof_cv = self.broof_classiefier()
        best_broof, best_params, train_time = self.randomized_search_fit(
            broof_cv, X_train, y_train)
        best_params_str = utils.dict_to_str(best_params)
        value_dict = self.predicting(best_broof, X_test, y_test)
        self.write_result_table_to_file('BROOF', best_params_str, value_dict,
                                        train_time)

        # XGBoost model training & testing
        xgb_cv = self.xgb_classifier()
        best_xgb, best_params, train_time = self.randomized_search_fit(
            xgb_cv, X_train, y_train)
        value_dict = self.predicting(best_xgb, X_test, y_test)
        self.write_result_table_to_file('XGBoost', best_params_str, value_dict,
                                        train_time)
Exemplo n.º 7
0
 def _ii(self, table: str, values: dict, conditions: dict = None):
     """
     :type table: str
     :type values: dict
     :type conditions: dict
     """
     cur = self.DB.cursor()
     if conditions is None:
         keys, values = dict_to_str(values)
         cur.execute(f"INSERT INTO {table} ({keys}) VALUES ({values})")
     else:
         # Update
         u_values = values.copy()
         for c in conditions:
             try:
                 u_values.pop(c)
             except KeyError:
                 pass
         # debug = f"UPDATE {table} SET {kcv(u_values, '=')} WHERE {kcv(conditions, '=', ' AND ')}"
         cur.execute(
             f"UPDATE {table} SET {kcv(u_values, '=')} WHERE {kcv(conditions, '=', ' AND ')}"
         )
Exemplo n.º 8
0
def add_hash(remain_dict, word_branch):
    '''Get a unique representation of remain_dict and add reference WordBranch.
    Args
        remain_dict ({char => int}) The remaining letters of the phrase from this point in the tree.
        word_branch (WordBranch)    The WordBranch we're working on.
    Returns
        (bool) Return True if the remain_dict was a new entry in the dictionary.
    '''
    hash_to_branch = WordBranch.get_hash_to_branch()

    dict_str = utils.dict_to_str(remain_dict)
    if dict_str in hash_to_branch:
        wb = hash_to_branch[dict_str]
        if wb.valid_children != None and len(wb.valid_children) > 0:

            mark_path(word_branch)
            word_branch.valid_children = wb.valid_children

        return False
    else:
        hash_to_branch[dict_str] = word_branch
        return True
Exemplo n.º 9
0
 def ii(self, table, vals, cur, conditions=None):
     """
     Inserts into the table vals using cur
     :param table: The table you're inserting into
     :param vals: The keys represent the titles and values the values inserted into said columns
     :param cur: The cursor used to communicate with the db
     :param conditions: The names of the columns you're checking in n,0 and their values in n,1
     """
     if conditions is None:
         keys, values = dict_to_str(vals)
         # debug = f"INSERT INTO {table} ({keys}) VALUES ({values})"
         cur.execute(f"INSERT INTO {table} ({keys}) VALUES ({values})")
     else:
         # Update
         u_vals = vals.copy()
         for c in conditions:
             try:
                 u_vals.pop(c)
             except KeyError:
                 pass
         # debug = f"UPDATE {table} SET {kcv(u_vals, '=')} WHERE {kcv(conditions, '=', ' AND ')}"
         cur.execute(
             f"UPDATE {table} SET {kcv(u_vals, '=')} WHERE {kcv(conditions, '=', ' AND ')}"
         )
Exemplo n.º 10
0
            self.set_custom_backup_props(backup)

            backup_doc = backup.to_document()
            get_mbs().backup_collection.save_document(backup_doc)
            # set the backup id from the saved doc

            backup.id = backup_doc["_id"]

            self.info("Saved backup \n%s" % backup)

            if backup.state == State.FAILED:
                trigger_task_finished_event(backup, State.FAILED)

            return backup
        except Exception, e:
            args_str = dict_to_str(kwargs)
            msg = ("Failed to schedule backup. Args:\n %s" % args_str)
            logger.error(msg)
            logger.error(traceback.format_exc())
            raise BackupSchedulingError(msg=msg, cause=e)

    ###########################################################################
    def set_custom_backup_props(self, backup):
        pass

    ###########################################################################
    def force_expire_backup(self, backup):
        self.backup_expiration_manager.expire_backup(backup, force=True)

    ###########################################################################
    def cancel_backup(self, backup):
Exemplo n.º 11
0
def test_dict_to_str():
    dict1 = {'p':1, 't':4, 'a':0, 'i':2}

    dict_str = utils.dict_to_str(dict1)
    assert dict_str == "pttttii"
Exemplo n.º 12
0
                                              (128, [2400, 8000], params_128)],
                                             desc='ImgSizes',
                                             leave=True,
                                             disable=args.disable_tqdm):

        for n_measure in tqdm(n_measures,
                              desc='N_measures',
                              leave=False,
                              disable=args.disable_tqdm):
            img_shape = (3, img_size, img_size)
            forward_model = GaussianCompressiveSensing(n_measure=n_measure,
                                                       img_shape=img_shape)
            # forward_model = NoOp()

            for img_name in tqdm(os.listdir(args.img_dir),
                                 desc='Images',
                                 leave=False,
                                 disable=args.disable_tqdm):
                orig_img = load_target_image(
                    os.path.join(args.img_dir, img_name), img_size).to(DEVICE)
                img_basename, _ = os.path.splitext(img_name)
                x_hat, x_degraded, _ = deep_decoder_recover(
                    orig_img,
                    forward_model,
                    run_dir='deep_decoder',
                    run_name=(img_basename + args.run_name_suffix + '.' +
                              dict_to_str(params) +
                              f'.n_measure={n_measure}.img_size={img_size}'),
                    img_size=img_size,
                    **params)
Exemplo n.º 13
0
def deep_decoder_images(args):
    if args.set_seed:
        torch.manual_seed(0)
        np.random.seed(0)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

    os.makedirs(BASE_DIR, exist_ok=True)

    metadata = recovery_settings[args.model]
    forwards = forward_models[args.model]

    data_split = Path(args.img_dir).name
    for img_name in tqdm(sorted(os.listdir(args.img_dir)),
                         desc='Images',
                         leave=True,
                         disable=args.disable_tqdm):
        orig_img = load_target_image(os.path.join(args.img_dir, img_name),
                                     metadata['img_size']).to(DEVICE)
        img_basename, _ = os.path.splitext(img_name)

        for f, f_args_list in tqdm(forwards.items(),
                                   desc='Forwards',
                                   leave=False,
                                   disable=args.disable_tqdm):
            for f_args in tqdm(f_args_list,
                               desc=f'{f} Args',
                               leave=False,
                               disable=args.disable_tqdm):
                f_args['img_shape'] = (3, metadata['img_size'],
                                       metadata['img_size'])
                forward_model = get_forward_model(f, **f_args)

                recovered_name = 'recovered.pt'
                results_folder = get_results_folder(
                    image_name=img_basename,
                    model=args.model,
                    n_cuts=0,  # NOTE - this field is unused for iagan
                    split=data_split,
                    forward_model=forward_model,
                    recovery_params=dict_to_str(metadata),
                    base_dir=BASE_DIR)

                os.makedirs(results_folder, exist_ok=True)

                recovered_path = results_folder / recovered_name
                if os.path.exists(recovered_path) and not args.overwrite:
                    print(f'{recovered_path} already exists, skipping...')
                    continue

                if args.run_name is not None:
                    current_run_name = (f'{img_basename}' +
                                        f'.{forward_model}' +
                                        dict_to_str(metadata) +
                                        f'.{args.run_name}')
                else:
                    current_run_name = None

                recovered_img, distorted_img, _ = deep_decoder_recover(
                    orig_img,
                    forward_model=forward_model,
                    optimizer=metadata['optimizer'],
                    num_filters=metadata['num_filters'],
                    depth=metadata['depth'],
                    lr=metadata['lr'],
                    img_size=metadata['img_size'],
                    steps=metadata['steps'],
                    restarts=metadata['restarts'],
                    run_dir=args.run_dir,
                    run_name=current_run_name,
                    disable_tqdm=args.disable_tqdm)

                # Make images folder
                img_folder = get_images_folder(split=data_split,
                                               image_name=img_basename,
                                               img_size=metadata['img_size'],
                                               base_dir=BASE_DIR)
                os.makedirs(img_folder, exist_ok=True)

                # Save original image if needed
                original_img_path = img_folder / 'original.pt'
                if not os.path.exists(original_img_path):
                    torch.save(orig_img, original_img_path)

                # Save distorted image if needed
                if forward_model.viewable:
                    distorted_img_path = img_folder / f'{forward_model}.pt'
                    if not os.path.exists(distorted_img_path):
                        torch.save(distorted_img, distorted_img_path)

                # Save recovered image and metadata
                torch.save(recovered_img, recovered_path)
                pickle.dump(metadata,
                            open(results_folder / 'metadata.pkl', 'wb'))
                p = psnr(recovered_img, orig_img)
                pickle.dump(p, open(results_folder / 'psnr.pkl', 'wb'))
Exemplo n.º 14
0
def mgan_images(args):
    if args.set_seed:
        torch.manual_seed(0)
        np.random.seed(0)
        torch.backends.cudnn.deterministic = True
        torch.backends.cudnn.benchmark = False

    os.makedirs(BASE_DIR, exist_ok=True)

    if args.model in ['mgan_began_cs']:
        gen = Generator128(64)
        gen = load_trained_net(
            gen, ('./checkpoints/celeba_began.withskips.bs32.cosine.min=0.25'
                  '.n_cuts=0/gen_ckpt.49.pt'))
        gen = gen.eval().to(DEVICE)
        img_size = 128
    elif args.model in ['mgan_vanilla_vae_cs']:
        gen = VAE()
        t = torch.load('./vae_checkpoints/vae_bs=128_beta=1.0/epoch_19.pt')
        gen.load_state_dict(t)
        gen = gen.eval().to(DEVICE)
        gen = gen.decoder
        img_size = 128
    elif args.model in ['mgan_dcgan_cs']:
        gen = dcgan_generator()
        t = torch.load(('./dcgan_checkpoints/netG.epoch_24.n_cuts_0.bs_64'
                        '.b1_0.5.lr_0.0002.pt'))
        gen.load_state_dict(t)
        gen = gen.eval().to(DEVICE)
        img_size = 64
    else:
        raise NotImplementedError()

    img_shape = (3, img_size, img_size)
    metadata = recovery_settings[args.model]
    n_cuts_list = metadata['n_cuts_list']
    del (metadata['n_cuts_list'])

    z_init_mode_list = metadata['z_init_mode']
    limit_list = metadata['limit']
    assert len(z_init_mode_list) == len(limit_list)
    del (metadata['z_init_mode'])
    del (metadata['limit'])

    forwards = forward_models[args.model]

    data_split = Path(args.img_dir).name
    for img_name in tqdm(sorted(os.listdir(args.img_dir)),
                         desc='Images',
                         leave=True,
                         disable=args.disable_tqdm):
        # Load image and get filename without extension
        orig_img = load_target_image(os.path.join(args.img_dir, img_name),
                                     img_size).to(DEVICE)
        img_basename, _ = os.path.splitext(img_name)

        for n_cuts in tqdm(n_cuts_list,
                           desc='N_cuts',
                           leave=False,
                           disable=args.disable_tqdm):
            metadata['n_cuts'] = n_cuts
            for i, (f, f_args_list) in enumerate(
                    tqdm(forwards.items(),
                         desc='Forwards',
                         leave=False,
                         disable=args.disable_tqdm)):
                for f_args in tqdm(f_args_list,
                                   desc=f'{f} Args',
                                   leave=False,
                                   disable=args.disable_tqdm):

                    f_args['img_shape'] = img_shape
                    forward_model = get_forward_model(f, **f_args)

                    for z_init_mode, limit in zip(
                            tqdm(z_init_mode_list,
                                 desc='z_init_mode',
                                 leave=False), limit_list):
                        metadata['z_init_mode'] = z_init_mode
                        metadata['limit'] = limit

                        # Before doing recovery, check if results already exist
                        # and possibly skip
                        recovered_name = 'recovered.pt'
                        results_folder = get_results_folder(
                            image_name=img_basename,
                            model=args.model,
                            n_cuts=n_cuts,
                            split=data_split,
                            forward_model=forward_model,
                            recovery_params=dict_to_str(metadata),
                            base_dir=BASE_DIR)

                        os.makedirs(results_folder, exist_ok=True)

                        recovered_path = results_folder / recovered_name
                        if os.path.exists(
                                recovered_path) and not args.overwrite:
                            print(
                                f'{recovered_path} already exists, skipping...'
                            )
                            continue

                        if args.run_name is not None:
                            current_run_name = (
                                f'{img_basename}.{forward_model}'
                                f'.{dict_to_str(metadata)}'
                                f'.{args.run_name}')
                        else:
                            current_run_name = None

                        recovered_img, distorted_img, _ = mgan_recover(
                            orig_img, gen, n_cuts, forward_model,
                            metadata['optimizer'], z_init_mode, limit,
                            metadata['z_lr'], metadata['n_steps'],
                            metadata['z_number'], metadata['restarts'],
                            args.run_dir, current_run_name, args.disable_tqdm)

                        # Make images folder
                        img_folder = get_images_folder(split=data_split,
                                                       image_name=img_basename,
                                                       img_size=img_size,
                                                       base_dir=BASE_DIR)
                        os.makedirs(img_folder, exist_ok=True)

                        # Save original image if needed
                        original_img_path = img_folder / 'original.pt'
                        if not os.path.exists(original_img_path):
                            torch.save(orig_img, original_img_path)

                        # Save distorted image if needed
                        if forward_model.viewable:
                            distorted_img_path = img_folder / f'{forward_model}.pt'
                            if not os.path.exists(distorted_img_path):
                                torch.save(distorted_img, distorted_img_path)

                        # Save recovered image and metadata
                        torch.save(recovered_img, recovered_path)
                        pickle.dump(
                            metadata,
                            open(results_folder / 'metadata.pkl', 'wb'))
                        p = psnr(recovered_img, orig_img)
                        pickle.dump(p, open(results_folder / 'psnr.pkl', 'wb'))