def _create_rl_train_net(self) -> None:
self.rl_train_model = ModelHelper(name="rl_train_" + self.model_id)
C2.set_model(self.rl_train_model)
if self.reward_shape is not None:
for action_index, boost in self.reward_shape.items():
action_boost = C2.Mul(
C2.Slice(
"actions", starts=[0, action_index], ends=[-1, action_index + 1]
),
boost,
broadcast=1,
)
C2.net().Sum(["rewards", action_boost], ["rewards"])
if self.maxq_learning:
next_q_values = self.get_max_q_values(
"next_states", self.get_possible_next_actions(), True
)
else:
next_q_values = self.get_q_values("next_states", "next_actions", True)
discount_blob = C2.ConstantFill("time_diff", value=self.rl_discount_rate)
if self.use_seq_num_diff_as_time_diff:
time_diff_adjusted_discount_blob = C2.Pow(
discount_blob, C2.Cast("time_diff", to=caffe2_pb2.TensorProto.FLOAT)
)
else:
time_diff_adjusted_discount_blob = discount_blob
q_vals_target = C2.Add(
"rewards",
C2.Mul(
C2.Mul(
C2.Cast(
"not_terminals", to=caffe2_pb2.TensorProto.FLOAT
), # type: ignore
time_diff_adjusted_discount_blob,
broadcast=1,
),
next_q_values,
),
)
self.update_model("states", "actions", q_vals_target)
workspace.RunNetOnce(self.rl_train_model.param_init_net)
self.rl_train_model.net.Proto().num_workers = (
RLTrainer.DEFAULT_TRAINING_NUM_WORKERS
)
self.rl_train_model.net.Proto().type = "async_scheduling"
workspace.CreateNet(self.rl_train_model.net)
C2.set_model(None)
def _create_rl_train_net(self) -> None:
self.rl_train_model = ModelHelper(name="rl_train_" + self.model_id)
C2.set_model(self.rl_train_model)
if self.maxq_learning:
next_q_values = self.get_max_q_values(
'next_states',
self.get_possible_next_actions(),
True,
)
else:
next_q_values = self.get_q_values('next_states', 'next_actions',
True)
discount_blob = C2.ConstantFill("time_diff",
value=self.rl_discount_rate)
time_diff_adjusted_discount_blob = C2.Pow(
discount_blob, C2.Cast("time_diff",
to=caffe2_pb2.TensorProto.FLOAT))
q_vals_target = C2.Add(
"rewards",
C2.Mul(
C2.Mul(
C2.Cast("not_terminals",
to=caffe2_pb2.TensorProto.FLOAT), # type: ignore
time_diff_adjusted_discount_blob,
broadcast=1,
),
next_q_values,
),
)
self.update_model('states', 'actions', q_vals_target)
workspace.RunNetOnce(self.rl_train_model.param_init_net)
self.rl_train_model.net.Proto().num_workers = \
RLTrainer.DEFAULT_TRAINING_NUM_WORKERS
workspace.CreateNet(self.rl_train_model.net)
C2.set_model(None)
def preprocess_blob(self, blob, normalization_parameters):
"""
Takes in a blob and its normalization parameters. Outputs a tuple
whose first element is a blob containing the normalized input blob
and whose second element contains all the parameter blobs used to
create it.
Call this from a CPU context and ensure the input blob exists in it.
"""
parameters: List[str] = []
ZERO = self._store_parameter(parameters, "ZERO",
np.array([0], dtype=np.float32))
ONE = self._store_parameter(parameters, "ONE",
np.array([1], dtype=np.float32))
NEGATIVE_ONE = self._store_parameter(parameters, "NEGATIVE_ONE",
np.array([-1], dtype=np.float32))
MISSING_U = self._store_parameter(
parameters, "MISSING_U",
np.array([MISSING_VALUE + 1e-4], dtype=np.float32))
MISSING_L = self._store_parameter(
parameters, "MISSING_L",
np.array([MISSING_VALUE - 1e-4], dtype=np.float32))
is_empty_l = C2.GT(blob, MISSING_L, broadcast=1)
is_empty_u = C2.LT(blob, MISSING_U, broadcast=1)
is_empty = C2.And(is_empty_l, is_empty_u)
for i in range(len(normalization_parameters) - 1):
if (normalization_parameters[i].feature_type !=
normalization_parameters[i + 1].feature_type):
raise Exception(
"Only one feature type is allowed per call to preprocess_blob!"
)
feature_type = normalization_parameters[0].feature_type
if feature_type == identify_types.BINARY:
TOLERANCE = self._store_parameter(parameters, "TOLERANCE",
np.array(1e-3, dtype=np.float32))
is_gt_zero = C2.GT(blob,
C2.Add(ZERO, TOLERANCE, broadcast=1),
broadcast=1)
is_lt_zero = C2.LT(blob,
C2.Sub(ZERO, TOLERANCE, broadcast=1),
broadcast=1)
bool_blob = C2.Or(is_gt_zero, is_lt_zero)
blob = C2.Cast(bool_blob, to=caffe2_pb2.TensorProto.FLOAT)
elif feature_type == identify_types.PROBABILITY:
clipped = C2.Clip(blob, min=0.01, max=0.99)
blob = C2.Mul(
C2.Log(C2.Sub(C2.Pow(clipped, exponent=-1.0), ONE,
broadcast=1)),
NEGATIVE_ONE,
broadcast=1,
)
elif feature_type == identify_types.ENUM:
for parameter in normalization_parameters:
possible_values = parameter.possible_values
for x in possible_values:
if x < 0:
logger.fatal(
"Invalid enum possible value for feature: " +
str(x) + " " + str(parameter.possible_values))
raise Exception(
"Invalid enum possible value for feature " + blob +
": " + str(x) + " " +
str(parameter.possible_values))
int_blob = C2.Cast(blob, to=core.DataType.INT32)
# Batch one hot transform with MISSING_VALUE as a possible value
feature_lengths = [
len(p.possible_values) + 1 for p in normalization_parameters
]
feature_lengths_blob = self._store_parameter(
parameters,
"feature_lengths_blob",
np.array(feature_lengths, dtype=np.int32),
)
feature_values = [
x for p in normalization_parameters
for x in p.possible_values + [int(MISSING_VALUE)]
]
feature_values_blob = self._store_parameter(
parameters,
"feature_values_blob",
np.array(feature_values, dtype=np.int32),
)
one_hot_output = C2.BatchOneHot(int_blob, feature_lengths_blob,
feature_values_blob)
flattened_one_hot = C2.FlattenToVec(one_hot_output)
# Remove missing values with a mask
cols_to_include = [[1] * len(p.possible_values) + [0]
for p in normalization_parameters]
cols_to_include = [x for col in cols_to_include for x in col]
mask = self._store_parameter(
parameters, "mask", np.array(cols_to_include, dtype=np.int32))
zero_vec = C2.ConstantFill(one_hot_output,
value=0,
dtype=caffe2_pb2.TensorProto.INT32)
repeated_mask_bool = C2.Cast(C2.Add(zero_vec, mask, broadcast=1),
to=core.DataType.BOOL)
flattened_repeated_mask = C2.FlattenToVec(repeated_mask_bool)
flattened_one_hot_proc = C2.NextBlob("flattened_one_hot_proc")
flattened_one_hot_proc_indices = C2.NextBlob(
"flattened_one_hot_proc_indices")
C2.net().BooleanMask(
[flattened_one_hot, flattened_repeated_mask],
[flattened_one_hot_proc, flattened_one_hot_proc_indices],
)
one_hot_shape = C2.Shape(one_hot_output)
shape_delta = self._store_parameter(
parameters,
"shape_delta",
np.array([0, len(normalization_parameters)], dtype=np.int64),
)
target_shape = C2.Sub(one_hot_shape, shape_delta, broadcast=1)
output_int_blob = C2.NextBlob("output_int_blob")
output_int_blob_old_shape = C2.NextBlob(
"output_int_blob_old_shape")
C2.net().Reshape(
[flattened_one_hot_proc, target_shape],
[output_int_blob, output_int_blob_old_shape],
)
output_blob = C2.Cast(output_int_blob, to=core.DataType.FLOAT)
return output_blob, parameters
elif feature_type == identify_types.QUANTILE:
# This transformation replaces a set of values with their quantile.
# The quantile boundaries are provided in the normalization params.
quantile_sizes = [
len(norm.quantiles) for norm in normalization_parameters
]
num_boundaries_blob = self._store_parameter(
parameters,
"num_boundaries_blob",
np.array(quantile_sizes, dtype=np.int32),
)
quantile_values = np.array([], dtype=np.float32)
quantile_labels = np.array([], dtype=np.float32)
for norm in normalization_parameters:
quantile_values = np.append(
quantile_values, np.array(norm.quantiles,
dtype=np.float32))
quantile_labels = np.append(
quantile_labels,
np.arange(len(norm.quantiles), dtype=np.float32) /
float(len(norm.quantiles) - 1.0),
)
quantiles = np.vstack([quantile_values, quantile_labels]).T
quantiles_blob = self._store_parameter(parameters,
"quantiles_blob", quantiles)
quantile_blob = C2.Percentile(blob, quantiles_blob,
num_boundaries_blob)
blob = quantile_blob
elif (feature_type == identify_types.CONTINUOUS
or feature_type == identify_types.BOXCOX):
boxcox_shifts = []
boxcox_lambdas = []
means = []
stddevs = []
for norm in normalization_parameters:
if feature_type == identify_types.BOXCOX:
assert (norm.boxcox_shift is not None
and norm.boxcox_lambda is not None)
boxcox_shifts.append(norm.boxcox_shift)
boxcox_lambdas.append(norm.boxcox_lambda)
means.append(norm.mean)
stddevs.append(norm.stddev)
if feature_type == identify_types.BOXCOX:
boxcox_shift_blob = self._store_parameter(
parameters,
"boxcox_shift",
np.array(boxcox_shifts, dtype=np.float32),
)
boxcox_lambda_blob = self._store_parameter(
parameters,
"boxcox_shift",
np.array(boxcox_lambdas, dtype=np.float32),
)
blob = C2.BatchBoxCox(blob, boxcox_lambda_blob,
boxcox_shift_blob)
means_blob = self._store_parameter(
parameters, "means_blob", np.array([means], dtype=np.float32))
stddevs_blob = self._store_parameter(
parameters, "stddevs_blob",
np.array([stddevs], dtype=np.float32))
blob = C2.Sub(blob, means_blob, broadcast=1, axis=0)
blob = C2.Div(blob, stddevs_blob, broadcast=1, axis=0)
if self.clip_anomalies:
blob = C2.Clip(blob, min=-5.0, max=5.0)
else:
raise NotImplementedError(
"Invalid feature type: {}".format(feature_type))
zeros = C2.ConstantFill(blob, value=0.0)
output_blob = C2.Where(is_empty, zeros, blob)
return output_blob, parameters
