def add_ops(self, net): """ Both the predict net and the eval net will call this function. For bootstrapping approach, the goal is to pass the cur_layer feature inputs through all the bootstrapped FCs that are stored under self.bootstrapped_FCs. Return the preds in the same output_schema with dummy indices (because they are not needed). """ version_info = get_current_scope().get( get_fc_predictor_version.__name__, {"fc_version": "fp32"}) predictor_fc_fp_version = version_info["fc_version"] for i in range(self.num_bootstrap): # these are dummy indices, not to be used anywhere indices = self._generate_bootstrapped_indices( net=net, copied_cur_layer=self.input_record.field_blobs()[0], iteration=i, ) params = self.bootstrapped_FCs[i * 2:(i * 2) + 2] self._add_ops( net=net, features=self.input_record, params=params, iteration=i, version=predictor_fc_fp_version, )
def _get_default_init_op(self): scale = math.sqrt(1.0 / self.input_dim) cur_scope = get_current_scope() trainer_version = get_sparse_lookup_trainer_version(**cur_scope.get( get_sparse_lookup_trainer_version.__name__, {'version': 'fp32'})) if trainer_version == 'fp32': default_weight_init = ('UniformFill', { 'min': -scale, 'max': scale }) elif trainer_version == 'fp16': default_weight_init = ("Float16UniformFill", { 'min': -scale, 'max': scale }) else: raise NotImplementedError( "Train version {} is not currently supported".format( trainer_version)) self.trainer_version = trainer_version return default_weight_init
def add_ops(self, net): version_info = get_current_scope().get( get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'} ) lookup_table_blob_size = self.shape[0] * self.shape[1] version = get_sparse_lookup_predictor_version( version_info['version'], blob_size=lookup_table_blob_size, min_blob_size_4bits=( version_info['min_blob_size_4bits'] if 'min_blob_size_4bits' in version_info else None ), embedding_dim=self.shape[1], sparse_feature_name=self.sparse_key, ) # TODO(amalevich): Layer should not be responsible for decision about # quantization. if not self.support_8bit() and version in {'uint8rowwise', 'fused_uint8rowwise', 'fused_uint4rowwise'}: version = 'fp16' self._add_ops(net, version, is_train=False)
def add_ops(self, net): """Both the predict net and the eval net will call this function """ version_info = get_current_scope().get( get_fc_predictor_version.__name__, {'fc_version': 'fp32'}) predictor_fc_fp_version = version_info['fc_version'] self._add_ops(net, self.param_blobs, predictor_fc_fp_version)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version(**cur_scope.get( get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) if _is_id_list(self.input_record): self._add_ops_id_list(net, version=version) elif _is_id_score_list(self.input_record): self._add_ops_id_score_list(net, version=version) else: raise "Unsupported input type {0}".format(self.input_record)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version( **cur_scope.get(get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) if _is_id_list(self.input_record): self._add_ops_id_list(net, version=version) elif _is_id_score_list(self.input_record): self._add_ops_id_score_list(net, version=version) else: raise "Unsupported input type {0}".format(self.input_record)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version(**cur_scope.get( get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) # TODO(amalevich): Layer should not be responsible for decision about # quantization. if not self.support_8bit() and version in { 'uint8rowwise', 'fused_uint8rowwise' }: version = 'fp32' self._add_ops(net, version)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version(**cur_scope.get( get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) if schema.equal_schemas(self.input_record, IdList): self._add_ops_id_list(net, version=version) elif schema.equal_schemas(self.input_record, IdScoreList, check_field_types=False): self._add_ops_id_score_list(net, version=version) else: raise "Unsupported input type {0}".format(self.input_record)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version( **cur_scope.get(get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) if schema.equal_schemas(self.input_record, IdList): self._add_ops_id_list(net, version=version) elif schema.equal_schemas(self.input_record, IdScoreList, check_field_types=False): self._add_ops_id_score_list(net, version=version) else: raise "Unsupported input type {0}".format(self.input_record)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version(**cur_scope.get( get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) # TODO(amalevich): Layer should not be responsible for decision about # quantization. if not self.support_8bit() and version in { 'uint8rowwise', 'fused_uint8rowwise' }: version = 'fp32' if _is_id_list(self.input_record): self._add_ops_id_list(net, version=version) elif _is_id_score_list(self.input_record): self._add_ops_id_score_list(net, version=version) else: raise "Unsupported input type {0}".format(self.input_record)
def add_ops(self, net): cur_scope = get_current_scope() version = get_sparse_lookup_predictor_version( **cur_scope.get(get_sparse_lookup_predictor_version.__name__, {'version': 'fp32'})) # TODO(amalevich): Layer should not be responsible for decision about # quantization. if not self.support_8bit() and version in {'uint8rowwise', 'fused_uint8rowwise'}: version = 'fp32' if _is_id_list(self.input_record): self._add_ops_id_list(net, version=version) elif _is_id_score_list(self.input_record): self._add_ops_id_score_list(net, version=version) else: raise "Unsupported input type {0}".format(self.input_record)
def __init__(self, model, input_record, inner_shape, reducer, weight_init=None, weight_optim=None, name='sparse_lookup', regularizer=None, **kwargs): super(SparseLookup, self).__init__(model, name, input_record, **kwargs) # TODO Add some asserts about input type if isinstance(inner_shape, int): inner_shape = [inner_shape] assert isinstance(inner_shape, list) or isinstance(inner_shape, tuple),\ "Unexpected type for inner_shape, expected list or tuple, got {0}".\ format(type(inner_shape)) if reducer == "PositionWeighted": assert _is_id_score_list(self.input_record), ( "PositionWeighted only support IdScoreList, but got {} " + "please use PositionWeighted layer to convert IdList " + "to IdScoreList").format(repr(self.input_record)) self.external_weights = input_record.values() elif reducer == "RecencyWeighted": assert _is_id_score_list(self.input_record), ( "RecencyWeighted only supports IdScoreList.") self.external_weights = input_record.values() self.reducer = reducer input_dim = get_categorical_limit(input_record) assert input_dim > 0, ( "{} should have categorical limit > 0, but got {}".format( get_key(input_record)(), input_dim)) self.input_dim = input_dim self.shape = [input_dim] + inner_shape cur_scope = get_current_scope() trainer_version = get_sparse_lookup_trainer_version(**cur_scope.get( get_sparse_lookup_trainer_version.__name__, {'version': 'fp32'})) self.trainer_version = trainer_version default_init_op = self._get_default_init_op() self.weight_init = weight_init or default_init_op if self.trainer_version == "fp16": assert self.weight_init[0] in self._fp16_compatible_init_op_types,\ "Fp16 training is enabled. Init op for weight parameter must be fp16"\ "compatibale. Got {}. Supported ops: {}".format( self.weight_init[0], self._fp16_compatible_init_op_types) if _is_id_list(self.input_record): sparse_key = self.input_record.items() elif _is_id_score_list(self.input_record): sparse_key = self.input_record.keys() else: raise NotImplementedError() if self.input_record.lengths.metadata: avg_length = self.input_record.lengths.metadata.expected_value else: avg_length = None self.w = self.create_param(param_name='w', shape=self.shape, initializer=self.weight_init, optimizer=weight_optim, ps_param=LayerPsParam( sparse_key=sparse_key, average_length=avg_length), regularizer=regularizer) self.scale_bias_init = ('ConstantFill', {'value': 0.0}) self.scale_bias = self.create_param( param_name='scale_bias', shape=[], initializer=self.scale_bias_init, optimizer=model.NoOptim, ) self.output_schema = schema.Scalar( (np.float32, inner_shape), self.get_next_blob_reference('output'), )