def test_fully_connected(self):
chooser = ValueNetBuilder__Union(
FullyConnected=value.fully_connected.FullyConnected())
builder = chooser.value
state_dim = 3
normalization_data = NormalizationData(
dense_normalization_parameters={
i: NormalizationParameters(feature_type=CONTINUOUS)
for i in range(state_dim)
})
value_network = builder.build_value_network(normalization_data)
batch_size = 5
x = torch.randn(batch_size, state_dim)
y = value_network(x)
self.assertEqual(y.shape, (batch_size, 1))
def _test_discrete_dqn_net_builder(
self,
chooser: DiscreteDQNNetBuilder__Union,
state_feature_config: Optional[rlt.ModelFeatureConfig] = None,
serving_module_class=DiscreteDqnPredictorWrapper,
) -> None:
builder = chooser.value
state_dim = 3
state_feature_config = state_feature_config or rlt.ModelFeatureConfig(
float_feature_infos=[
rlt.FloatFeatureInfo(name=f"f{i}", feature_id=i)
for i in range(state_dim)
]
)
state_dim = len(state_feature_config.float_feature_infos)
state_normalization_data = NormalizationData(
dense_normalization_parameters={
fi.feature_id: NormalizationParameters(
feature_type=CONTINUOUS, mean=0.0, stddev=1.0
)
for fi in state_feature_config.float_feature_infos
}
)
action_names = ["L", "R"]
q_network = builder.build_q_network(
state_feature_config, state_normalization_data, len(action_names)
)
x = q_network.input_prototype()
y = q_network(x)
self.assertEqual(y.shape, (1, 2))
serving_module = builder.build_serving_module(
q_network, state_normalization_data, action_names, state_feature_config
)
self.assertIsInstance(serving_module, serving_module_class)
def identify_parameter(
feature_name,
values,
max_unique_enum_values=DEFAULT_MAX_UNIQUE_ENUM,
quantile_size=DEFAULT_MAX_QUANTILE_SIZE,
quantile_k2_threshold=DEFAULT_QUANTILE_K2_THRESHOLD,
skip_box_cox=False,
skip_quantiles=False,
feature_type=None,
):
if feature_type is None:
feature_type = identify_types.identify_type(values, max_unique_enum_values)
boxcox_lambda = None
boxcox_shift = 0.0
mean = 0.0
stddev = 1.0
possible_values = None
quantiles = None
assert feature_type in [
identify_types.CONTINUOUS,
identify_types.PROBABILITY,
identify_types.BINARY,
identify_types.ENUM,
identify_types.CONTINUOUS_ACTION,
identify_types.DO_NOT_PREPROCESS,
], "unknown type {}".format(feature_type)
assert (
len(values) >= MINIMUM_SAMPLES_TO_IDENTIFY
), "insufficient information to identify parameter"
min_value = float(np.min(values))
max_value = float(np.max(values))
if feature_type == identify_types.DO_NOT_PREPROCESS:
mean = float(np.mean(values))
values = values - mean
stddev = max(float(np.std(values, ddof=1)), 1.0)
if feature_type == identify_types.CONTINUOUS:
if min_value == max_value:
return no_op_feature()
k2_original, p_original = stats.normaltest(values)
# shift can be estimated but not in scipy
boxcox_shift = float(min_value * -1)
candidate_values, lambda_ = stats.boxcox(
np.maximum(values + boxcox_shift, BOX_COX_MARGIN)
)
k2_boxcox, p_boxcox = stats.normaltest(candidate_values)
logger.info(
"Feature stats. Original K2: {} P: {} Boxcox K2: {} P: {}".format(
k2_original, p_original, k2_boxcox, p_boxcox
)
)
if lambda_ < 0.9 or lambda_ > 1.1:
# Lambda is far enough from 1.0 to be worth doing boxcox
if k2_original > k2_boxcox * 10 and k2_boxcox <= quantile_k2_threshold:
# The boxcox output is significantly more normally distributed
# than the original data and is normal enough to apply
# effectively.
stddev = float(np.std(candidate_values, ddof=1))
# Unclear whether this happens in practice or not
if (
np.isfinite(stddev)
and stddev < BOX_COX_MAX_STDDEV
and not np.isclose(stddev, 0)
):
values = candidate_values
boxcox_lambda = float(lambda_)
if boxcox_lambda is None or skip_box_cox:
boxcox_shift = None
boxcox_lambda = None
if boxcox_lambda is not None:
feature_type = identify_types.BOXCOX
if (
boxcox_lambda is None
and k2_original > quantile_k2_threshold
and (not skip_quantiles)
):
feature_type = identify_types.QUANTILE
quantiles = (
np.unique(
mquantiles(
values,
np.arange(quantile_size + 1, dtype=np.float64)
/ float(quantile_size),
alphap=0.0,
betap=1.0,
)
)
.astype(float)
.tolist()
)
logger.info("Feature is non-normal, using quantiles: {}".format(quantiles))
if (
feature_type == identify_types.CONTINUOUS
or feature_type == identify_types.BOXCOX
or feature_type == identify_types.CONTINUOUS_ACTION
):
mean = float(np.mean(values))
values = values - mean
stddev = max(float(np.std(values, ddof=1)), 1.0)
if not np.isfinite(stddev):
logger.info("Std. dev not finite for feature {}".format(feature_name))
return None
values /= stddev
if feature_type == identify_types.ENUM:
possible_values = np.unique(values.astype(int)).astype(int).tolist()
return NormalizationParameters(
feature_type,
boxcox_lambda,
boxcox_shift,
mean,
stddev,
possible_values,
quantiles,
min_value,
max_value,
)
def no_op_feature():
return NormalizationParameters(
identify_types.CONTINUOUS, None, 0, 0, 1, None, None, None, None
)
def action_normalization_data(self) -> NormalizationData:
return NormalizationData(
dense_normalization_parameters={
i: NormalizationParameters(feature_type="DISCRETE_ACTION")
for i in range(len(self.action_names))
})
def test_seq2slate_scriptable(self):
state_dim = 2
candidate_dim = 3
num_stacked_layers = 2
num_heads = 2
dim_model = 128
dim_feedforward = 128
candidate_size = 8
slate_size = 8
output_arch = Seq2SlateOutputArch.AUTOREGRESSIVE
temperature = 1.0
greedy_serving = True
# test the raw Seq2Slate model is script-able
seq2slate = Seq2SlateTransformerModel(
state_dim=state_dim,
candidate_dim=candidate_dim,
num_stacked_layers=num_stacked_layers,
num_heads=num_heads,
dim_model=dim_model,
dim_feedforward=dim_feedforward,
max_src_seq_len=candidate_size,
max_tgt_seq_len=slate_size,
output_arch=output_arch,
temperature=temperature,
)
seq2slate_scripted = torch.jit.script(seq2slate)
seq2slate_net = Seq2SlateTransformerNet(
state_dim=state_dim,
candidate_dim=candidate_dim,
num_stacked_layers=num_stacked_layers,
num_heads=num_heads,
dim_model=dim_model,
dim_feedforward=dim_feedforward,
max_src_seq_len=candidate_size,
max_tgt_seq_len=slate_size,
output_arch=output_arch,
temperature=temperature,
)
state_normalization_data = NormalizationData(
dense_normalization_parameters={
0: NormalizationParameters(feature_type=DO_NOT_PREPROCESS),
1: NormalizationParameters(feature_type=DO_NOT_PREPROCESS),
})
candidate_normalization_data = NormalizationData(
dense_normalization_parameters={
5: NormalizationParameters(feature_type=DO_NOT_PREPROCESS),
6: NormalizationParameters(feature_type=DO_NOT_PREPROCESS),
7: NormalizationParameters(feature_type=DO_NOT_PREPROCESS),
})
state_preprocessor = Preprocessor(
state_normalization_data.dense_normalization_parameters, False)
candidate_preprocessor = Preprocessor(
candidate_normalization_data.dense_normalization_parameters, False)
# test seq2slate with preprocessor is scriptable
seq2slate_with_preprocessor = Seq2SlateWithPreprocessor(
seq2slate_net.eval(),
state_preprocessor,
candidate_preprocessor,
greedy_serving,
)
torch.jit.script(seq2slate_with_preprocessor)
