def set_request_only(field): for f in field.all_scalars(): categorical_limit, expected_value = None, None if not f.metadata: feature_specs = schema.FeatureSpec( feature_is_request_only=True, ) elif not f.metadata.feature_specs: categorical_limit = f.metadata.categorical_limit expected_value = f.metadata.expected_value feature_specs = schema.FeatureSpec( feature_is_request_only=True, ) else: categorical_limit = f.metadata.categorical_limit expected_value = f.metadata.expected_value feature_specs = schema.FeatureSpec( feature_type=f.metadata.feature_specs.feature_type, feature_names=f.metadata.feature_specs.feature_names, feature_ids=f.metadata.feature_specs.feature_ids, feature_is_request_only=True, ) # make sure not to set categorical_limit for a non-integer field if not np.issubdtype(f.field_type(), np.integer): assert categorical_limit is None, \ "categorical_limit shouldn't be set for no-integer field" f.set_metadata( schema.Metadata( categorical_limit=categorical_limit, expected_value=expected_value, feature_specs=feature_specs, ) )
def __init__(self, model, input_record, axis=1, add_axis=0, name='concat', **kwargs): super(Concat, self).__init__(model, name, input_record, **kwargs) self.axis = axis self.add_axis = add_axis assert not (axis == 0 and add_axis == 1), \ "It's not allowed to add axis=0" assert isinstance(input_record, schema.Struct),\ "Incorrect input type. Expected Struct, but received: {0}".\ format(input_record) shapes = [] for field_name, field_type in viewitems(input_record.fields): assert isinstance(field_type, schema.Scalar),\ "Incorrect input type for {}. Expected Scalar, but got: {}".\ format(field_name, field_type) # Assume that first dimension is batch, so actual axis in shape is # axis - 1 shape = list(field_type.field_type().shape) if add_axis: shape.insert(axis - 1, 1) assert len(shape) >= axis,\ "Concat expects that limited dimensions of the input tensor" shapes.append(shape) logger.info('Concat Layer input shapes: ' + str(shapes)) if axis == 0: self.output_schema = schema.from_blob_list( input_record[0], [self.get_next_blob_reference('output')]) return concat_dim = 0 for shape in shapes: concat_dim += shape[axis - 1] shape[axis - 1] = 0 assert shape == shapes[0],\ "Shapes {0} and {1} are not compatible for Concat".\ format(shape, shapes[0]) output_dims = shapes[0] output_dims[axis - 1] = concat_dim logger.info('Concat Layer output_dims: ' + str(output_dims)) self.output_schema = schema.Scalar( (np.float32, output_dims), self.get_next_blob_reference('output')) record_to_concat = input_record.fields.values() concated_feature_to_index = get_concatenated_feature_to_index( record_to_concat) if concated_feature_to_index: metadata = schema.Metadata(feature_specs=schema.FeatureSpec( feature_to_index=concated_feature_to_index)) self.output_schema.set_metadata(metadata)
def testSetRequestOnly(self): input_record = schema.Scalar(np.int64) schema.attach_metadata_to_scalars( input_record, schema.Metadata( categorical_limit=100000000, expected_value=99, feature_specs=schema.FeatureSpec(feature_ids=[1, 100, 1001]))) set_request_only(input_record) self.assertEqual(input_record.metadata.categorical_limit, 100000000) self.assertEqual(input_record.metadata.expected_value, 99) self.assertEqual(input_record.metadata.feature_specs.feature_ids, [1, 100, 1001])
def __init__(self, model, input_record, inner_shape, reducer, weight_init=None, weight_optim=None, name='sparse_lookup', **kwargs): super(SparseLookup, self).__init__(model, name, input_record, **kwargs) if isinstance(inner_shape, int): inner_shape = [inner_shape] assert isinstance(inner_shape, list) or isinstance(inner_shape, tuple),\ "Unexpected type for inner_shape, expected list or tuple, got {0}".\ format(type(inner_shape)) # TODO Add some asserts about input type assert reducer in self._supported_reducers, "Unsupported reducer: {}".\ format(reducer) self.reducer = reducer assert input_record.items.metadata is not None,\ "Features without metadata are not supported" input_dim = input_record.items.metadata.categorical_limit assert input_dim is not None, "Unbounded features are not supported" self.output_schema = schema.Scalar( (np.float32, inner_shape), model.net.NextScopedBlob(name + '_output'), ) if self.request_only: schema.attach_metadata_to_scalars( self.output_schema, schema.Metadata(categorical_limit=None, expected_value=None, feature_specs=schema.FeatureSpec( feature_is_request_only=True))) scale = math.sqrt(1.0 / input_dim) self.shape = [input_dim] + inner_shape self.weight_init = weight_init if weight_init else ('UniformFill', { 'min': -scale, 'max': scale }) self.w = model.net.NextScopedBlob(name + "_w") self.params.append( LayerParameter(parameter=self.w, initializer=core.CreateOperator( self.weight_init[0], [], self.w, shape=self.shape, **self.weight_init[1]), optimizer=weight_optim)) if reducer == 'PositionWeighted': self.pos_w = model.net.NextScopedBlob(name + "_pos_w") self.params.append( LayerParameter(parameter=self.pos_w, initializer=core.CreateOperator('ConstantFill', [], self.pos_w, shape=[ input_dim, ], value=1.0), optimizer=weight_optim))