def test_fully_connected(self): chooser = ValueNetBuilder__Union( FullyConnected=value.fully_connected.FullyConnected()) builder = chooser.value state_dim = 3 normalization_data = NormalizationData( dense_normalization_parameters={ i: NormalizationParameters(feature_type=CONTINUOUS) for i in range(state_dim) }) value_network = builder.build_value_network(normalization_data) batch_size = 5 x = torch.randn(batch_size, state_dim) y = value_network(x) self.assertEqual(y.shape, (batch_size, 1))
def _test_discrete_dqn_net_builder( self, chooser: DiscreteDQNNetBuilder__Union, state_feature_config: Optional[rlt.ModelFeatureConfig] = None, serving_module_class=DiscreteDqnPredictorWrapper, ) -> None: builder = chooser.value state_dim = 3 state_feature_config = state_feature_config or rlt.ModelFeatureConfig( float_feature_infos=[ rlt.FloatFeatureInfo(name=f"f{i}", feature_id=i) for i in range(state_dim) ] ) state_dim = len(state_feature_config.float_feature_infos) state_normalization_data = NormalizationData( dense_normalization_parameters={ fi.feature_id: NormalizationParameters( feature_type=CONTINUOUS, mean=0.0, stddev=1.0 ) for fi in state_feature_config.float_feature_infos } ) action_names = ["L", "R"] q_network = builder.build_q_network( state_feature_config, state_normalization_data, len(action_names) ) x = q_network.input_prototype() y = q_network(x) self.assertEqual(y.shape, (1, 2)) serving_module = builder.build_serving_module( q_network, state_normalization_data, action_names, state_feature_config ) self.assertIsInstance(serving_module, serving_module_class)
def identify_parameter( feature_name, values, max_unique_enum_values=DEFAULT_MAX_UNIQUE_ENUM, quantile_size=DEFAULT_MAX_QUANTILE_SIZE, quantile_k2_threshold=DEFAULT_QUANTILE_K2_THRESHOLD, skip_box_cox=False, skip_quantiles=False, feature_type=None, ): if feature_type is None: feature_type = identify_types.identify_type(values, max_unique_enum_values) boxcox_lambda = None boxcox_shift = 0.0 mean = 0.0 stddev = 1.0 possible_values = None quantiles = None assert feature_type in [ identify_types.CONTINUOUS, identify_types.PROBABILITY, identify_types.BINARY, identify_types.ENUM, identify_types.CONTINUOUS_ACTION, identify_types.DO_NOT_PREPROCESS, ], "unknown type {}".format(feature_type) assert ( len(values) >= MINIMUM_SAMPLES_TO_IDENTIFY ), "insufficient information to identify parameter" min_value = float(np.min(values)) max_value = float(np.max(values)) if feature_type == identify_types.DO_NOT_PREPROCESS: mean = float(np.mean(values)) values = values - mean stddev = max(float(np.std(values, ddof=1)), 1.0) if feature_type == identify_types.CONTINUOUS: if min_value == max_value: return no_op_feature() k2_original, p_original = stats.normaltest(values) # shift can be estimated but not in scipy boxcox_shift = float(min_value * -1) candidate_values, lambda_ = stats.boxcox( np.maximum(values + boxcox_shift, BOX_COX_MARGIN) ) k2_boxcox, p_boxcox = stats.normaltest(candidate_values) logger.info( "Feature stats. Original K2: {} P: {} Boxcox K2: {} P: {}".format( k2_original, p_original, k2_boxcox, p_boxcox ) ) if lambda_ < 0.9 or lambda_ > 1.1: # Lambda is far enough from 1.0 to be worth doing boxcox if k2_original > k2_boxcox * 10 and k2_boxcox <= quantile_k2_threshold: # The boxcox output is significantly more normally distributed # than the original data and is normal enough to apply # effectively. stddev = float(np.std(candidate_values, ddof=1)) # Unclear whether this happens in practice or not if ( np.isfinite(stddev) and stddev < BOX_COX_MAX_STDDEV and not np.isclose(stddev, 0) ): values = candidate_values boxcox_lambda = float(lambda_) if boxcox_lambda is None or skip_box_cox: boxcox_shift = None boxcox_lambda = None if boxcox_lambda is not None: feature_type = identify_types.BOXCOX if ( boxcox_lambda is None and k2_original > quantile_k2_threshold and (not skip_quantiles) ): feature_type = identify_types.QUANTILE quantiles = ( np.unique( mquantiles( values, np.arange(quantile_size + 1, dtype=np.float64) / float(quantile_size), alphap=0.0, betap=1.0, ) ) .astype(float) .tolist() ) logger.info("Feature is non-normal, using quantiles: {}".format(quantiles)) if ( feature_type == identify_types.CONTINUOUS or feature_type == identify_types.BOXCOX or feature_type == identify_types.CONTINUOUS_ACTION ): mean = float(np.mean(values)) values = values - mean stddev = max(float(np.std(values, ddof=1)), 1.0) if not np.isfinite(stddev): logger.info("Std. dev not finite for feature {}".format(feature_name)) return None values /= stddev if feature_type == identify_types.ENUM: possible_values = np.unique(values.astype(int)).astype(int).tolist() return NormalizationParameters( feature_type, boxcox_lambda, boxcox_shift, mean, stddev, possible_values, quantiles, min_value, max_value, )
def no_op_feature(): return NormalizationParameters( identify_types.CONTINUOUS, None, 0, 0, 1, None, None, None, None )
def action_normalization_data(self) -> NormalizationData: return NormalizationData( dense_normalization_parameters={ i: NormalizationParameters(feature_type="DISCRETE_ACTION") for i in range(len(self.action_names)) })
def test_seq2slate_scriptable(self): state_dim = 2 candidate_dim = 3 num_stacked_layers = 2 num_heads = 2 dim_model = 128 dim_feedforward = 128 candidate_size = 8 slate_size = 8 output_arch = Seq2SlateOutputArch.AUTOREGRESSIVE temperature = 1.0 greedy_serving = True # test the raw Seq2Slate model is script-able seq2slate = Seq2SlateTransformerModel( state_dim=state_dim, candidate_dim=candidate_dim, num_stacked_layers=num_stacked_layers, num_heads=num_heads, dim_model=dim_model, dim_feedforward=dim_feedforward, max_src_seq_len=candidate_size, max_tgt_seq_len=slate_size, output_arch=output_arch, temperature=temperature, ) seq2slate_scripted = torch.jit.script(seq2slate) seq2slate_net = Seq2SlateTransformerNet( state_dim=state_dim, candidate_dim=candidate_dim, num_stacked_layers=num_stacked_layers, num_heads=num_heads, dim_model=dim_model, dim_feedforward=dim_feedforward, max_src_seq_len=candidate_size, max_tgt_seq_len=slate_size, output_arch=output_arch, temperature=temperature, ) state_normalization_data = NormalizationData( dense_normalization_parameters={ 0: NormalizationParameters(feature_type=DO_NOT_PREPROCESS), 1: NormalizationParameters(feature_type=DO_NOT_PREPROCESS), }) candidate_normalization_data = NormalizationData( dense_normalization_parameters={ 5: NormalizationParameters(feature_type=DO_NOT_PREPROCESS), 6: NormalizationParameters(feature_type=DO_NOT_PREPROCESS), 7: NormalizationParameters(feature_type=DO_NOT_PREPROCESS), }) state_preprocessor = Preprocessor( state_normalization_data.dense_normalization_parameters, False) candidate_preprocessor = Preprocessor( candidate_normalization_data.dense_normalization_parameters, False) # test seq2slate with preprocessor is scriptable seq2slate_with_preprocessor = Seq2SlateWithPreprocessor( seq2slate_net.eval(), state_preprocessor, candidate_preprocessor, greedy_serving, ) torch.jit.script(seq2slate_with_preprocessor)