示例#1
0
    def next(self):
        next_index = self._subset_iterator.next()
        if isinstance(next_index, numpy.ndarray) and len(next_index) == 1:
            next_index = next_index[0]
        self._needs_cast = True

        if self._needs_cast:
            features = numpy.cast[config.floatX](self._raw_data[0][next_index])
        else:
            features = self._raw_data[0][next_index, :]
        #import ipdb; ipdb.set_trace()
        if self.topo:
            if len(features.shape) != 2:
                features = features.reshape((1, features.shape[0]))
            features = self._dataset.get_topological_view(features)
        if self.targets:
            bbx_targets = get_image_bboxes(next_index, self._raw_data[1])
            if len(bbx_targets.shape) != 2:
                bbx_targets = bbx_targets.reshape((1, bbx_targets.shape[0]))

            if self.use_output_map:
                if self.bbox_conversion_type == ConversionType.GUID:
                    if isinstance(self._data_specs[0].components[1],
                                  Conv2DSpace):
                        conv_outs = True
                    else:
                        conv_outs = False
                    n_channels = self._data_specs[0].components[1].num_channels
                    targets = convert_bboxes_guided(bbx_targets,
                                                    self.img_shape,
                                                    self.receptive_field_shape,
                                                    self.area_ratio,
                                                    self.stride,
                                                    conv_outs=conv_outs,
                                                    n_channels=n_channels)
                else:
                    targets = convert_bboxes_exhaustive(
                        bbx_targets, self.img_shape,
                        self.receptive_field_shape, self.area_ratio,
                        self.stride)
            else:
                targets = self.get_bare_outputs(bbx_targets)

            if targets.shape[0] != features.shape[0]:
                raise ValueError(
                    "There is a batch size mismatch between features and targets."
                )

            self._targets_need_cast = True
            if self._targets_need_cast:
                targets = numpy.cast[config.floatX](targets)
            return features, targets
        else:
            if self._return_tuple:
                features = (features, )
            return features
示例#2
0
    def resize(h5file, start, stop, remove_old_node=False):
        if h5file is None:
            raise ValueError("h5file should not be None.")

        data = h5file.root.Data
        node_name = "Data_%s_%s" % (start, stop)
        if remove_old_node:
            try:
                gcolumns = h5file.createGroup('/', node_name,
                                              "Data %s" % node_name)
            except tables.exceptions.NodeError:
                h5file.removeNode('/', node_name, 1)
                gcolumns = h5file.createGroup('/', node_name,
                                              "Data %s" % node_name)
        elif node_name in h5file.root:
            return h5file, getattr(h5file.root, node_name)
        else:
            gcolumns = h5file.createGroup('/', node_name,
                                          "Data %s" % node_name)

        if FaceBBoxDDMPytables.h5file is None:
            FaceBBoxDDMPytables.h5file = h5file

        start = 0 if start is None else start
        stop = gcolumns.X.nrows if stop is None else stop

        atom = tables.Float32Atom(
        ) if config.floatX == 'float32' else tables.Float64Atom()
        filters = FaceBBoxDDMPytables.filters

        x = h5file.createCArray(gcolumns,
                                'X',
                                atom=atom,
                                shape=((stop - start, data.X.shape[1])),
                                title="Images",
                                filters=filters)

        y = h5file.createTable(gcolumns,
                               'bboxes',
                               BoundingBox,
                               title="Face bounding boxes",
                               filters=filters)

        x[:] = data.X[start:stop]
        bboxes = get_image_bboxes(slice(start, stop), data.bboxes)
        y.append(bboxes)

        if remove_old_node:
            h5file.removeNode('/', "Data", 1)
            h5file.renameNode('/', "Data", node_name)

        h5file.flush()
        return h5file, gcolumns
示例#3
0
    def next(self):
        next_index = self._subset_iterator.next()
        if isinstance(next_index, numpy.ndarray) and len(next_index) == 1:
            next_index = next_index[0]
        self._needs_cast = True

        if self._needs_cast:
            features = numpy.cast[config.floatX](self._raw_data[0][next_index])
        else:
            features = self._raw_data[0][next_index,:]
        #import ipdb; ipdb.set_trace()
        if self.topo:
            if len(features.shape) != 2:
                features = features.reshape((1, features.shape[0]))
            features = self._dataset.get_topological_view(features)
        if self.targets:
            bbx_targets = get_image_bboxes(next_index, self._raw_data[1])
            if len(bbx_targets.shape) != 2:
                bbx_targets = bbx_targets.reshape((1, bbx_targets.shape[0]))

            if self.use_output_map:
                if self.bbox_conversion_type == ConversionType.GUID:
                    if isinstance(self._data_specs[0].components[1], Conv2DSpace):
                        conv_outs = True
                    else:
                        conv_outs = False
                    n_channels = self._data_specs[0].components[1].num_channels
                    targets = convert_bboxes_guided(bbx_targets, self.img_shape,
                            self.receptive_field_shape, self.area_ratio, self.stride,
                            conv_outs=conv_outs, n_channels=n_channels)
                else:
                    targets = convert_bboxes_exhaustive(bbx_targets, self.img_shape,
                            self.receptive_field_shape, self.area_ratio, self.stride)
            else:
                targets = self.get_bare_outputs(bbx_targets)

            if targets.shape[0] != features.shape[0]:
                raise ValueError("There is a batch size mismatch between features and targets.")

            self._targets_need_cast = True
            if self._targets_need_cast:
                targets = numpy.cast[config.floatX](targets)
            return features, targets
        else:
            if self._return_tuple:
                features = (features,)
            return features
示例#4
0
    def resize(h5file, start, stop, remove_old_node=False):
        if h5file is None:
            raise ValueError("h5file should not be None.")

        data = h5file.root.Data
        node_name = "Data_%s_%s" % (start, stop)
        if remove_old_node:
            try:
                gcolumns = h5file.createGroup('/', node_name, "Data %s" %   node_name)
            except tables.exceptions.NodeError:
                h5file.removeNode('/', node_name, 1)
                gcolumns = h5file.createGroup('/', node_name, "Data %s" % node_name)
        elif node_name in h5file.root:
            return h5file, getattr(h5file.root, node_name)
        else:
            gcolumns = h5file.createGroup('/', node_name, "Data %s" %   node_name)

        if FaceBBoxDDMPytables.h5file is None:
            FaceBBoxDDMPytables.h5file = h5file

        start = 0 if start is None else start
        stop = gcolumns.X.nrows if stop is None else stop

        atom = tables.Float32Atom() if config.floatX == 'float32' else tables.Float64Atom()
        filters = FaceBBoxDDMPytables.filters

        x = h5file.createCArray(gcolumns, 'X', atom = atom, shape = ((stop - start, data.X.shape[1])),
                title = "Images", filters = filters)

        y = h5file.createTable(gcolumns, 'bboxes', BoundingBox,
                title = "Face bounding boxes", filters = filters)

        x[:] = data.X[start:stop]
        bboxes = get_image_bboxes(slice(start, stop), data.bboxes)
        y.append(bboxes)

        if remove_old_node:
            h5file.removeNode('/', "Data", 1)
            h5file.renameNode('/', "Data", node_name)

        h5file.flush()
        return h5file, gcolumns
示例#5
0
    def apply(self,
                dataset,
                start=None,
                stop=None,
                is_face=True,
                can_fit=False):
        """
        is_face: Bool,
            Flag that determines if we operate on face or nonface dataset.
        """

        if is_face:
            X = dataset.face_dataset.get_topological_view()
        else:
            X = dataset.nonface_dataset.get_topological_view()

        num_topological_dimensions = len(X.shape) - 2

        if num_topological_dimensions != len(self.patch_shape):
            raise ValueError("ExtractGridPatches with "
                             + str(len(self.patch_shape))
                             + " topological dimensions called on"
                             + " dataset with " +
                             str(num_topological_dimensions) + ".")

        num_patches = X.shape[0]
        max_strides = [X.shape[0] - 1]
        for i in xrange(num_topological_dimensions):
            patch_width = self.patch_shape[i]
            data_width = X.shape[i + 1]
            last_valid_coord = data_width - patch_width
            if last_valid_coord < 0:
                raise ValueError('On topological dimension ' + str(i) +
                                 ', the data has width ' + str(data_width) +
                                 ' but the requested patch width is ' +
                                 str(patch_width))

            stride = self.patch_stride[i]
            if stride == 0:
                max_stride_this_axis = 0
            else:
                max_stride_this_axis = last_valid_coord / stride
            num_strides_this_axis = max_stride_this_axis + 1
            max_strides.append(max_stride_this_axis)
            num_patches *= num_strides_this_axis

        # batch size
        output_shape = [num_patches]

        # topological dimensions
        for dim in self.patch_shape:
            output_shape.append(dim)

        # number of channels
        output_shape.append(X.shape[-1])
        output = numpy.zeros(output_shape, dtype=X.dtype)
        channel_slice = slice(0, X.shape[-1])
        coords = [0] * (num_topological_dimensions + 1)
        keep_going = True
        i = 0

        bboxes = dataset.face_dataset.bboxes

        while keep_going:

            args = [coords[0]]

            for j in xrange(num_topological_dimensions):
                coord = coords[j + 1] * self.patch_stride[j]
                args.append(slice(coord, coord + self.patch_shape[j]))

            args.append(channel_slice)
            patch = X[args]
            output[i, :] = patch
            i += 1

            # increment coordinates
            j = 0
            keep_going = False

            while not keep_going:
                if coords[-(j + 1)] < max_strides[-(j + 1)]:
                    coords[-(j + 1)] += 1
                    keep_going = True
                else:
                    coords[-(j + 1)] = 0
                    if j == num_topological_dimensions:
                        break
                    j = j + 1

        portion = slice(start, stop)

        repeat_times = numpy.ones(X.shape[0], 1)
        if is_face:
            bbox_targets = get_image_bboxes(portion, bboxes)

            outputmaps = convert_bboxes_guided(bbox_targets, (X.shape[0], X.shape[1]),
                self.patch_shape,
                area_ratio=self.area_ratio,
                stride=stride)
        else:
            targets = numpy.zeros(output_shape[-1])
            outputmaps = repeat_times * targets

        patch_loc = numpy.arange(output_shape[-1])
        patch_locs = repeat_times * patch_loc

        if start is None and stop is None:
            return output, outputmaps.flatten(), patch_locs.flatten()
        elif start is None:
            raise ValueError("You should give a start value.")
        elif start is not None and stop is None:
            return output[start], outputmaps.flatten(), patch_locs[start].flatten()
        else:
            return output[start:stop], outputmaps.flatten(), patch_locs[start:stop].flatten()
示例#6
0
    def apply(self,
              dataset,
              start=None,
              stop=None,
              is_face=True,
              can_fit=False):
        """
        is_face: Bool,
            Flag that determines if we operate on face or nonface dataset.
        """

        if is_face:
            X = dataset.face_dataset.get_topological_view()
        else:
            X = dataset.nonface_dataset.get_topological_view()

        num_topological_dimensions = len(X.shape) - 2

        if num_topological_dimensions != len(self.patch_shape):
            raise ValueError("ExtractGridPatches with " +
                             str(len(self.patch_shape)) +
                             " topological dimensions called on" +
                             " dataset with " +
                             str(num_topological_dimensions) + ".")

        num_patches = X.shape[0]
        max_strides = [X.shape[0] - 1]
        for i in xrange(num_topological_dimensions):
            patch_width = self.patch_shape[i]
            data_width = X.shape[i + 1]
            last_valid_coord = data_width - patch_width
            if last_valid_coord < 0:
                raise ValueError('On topological dimension ' + str(i) +
                                 ', the data has width ' + str(data_width) +
                                 ' but the requested patch width is ' +
                                 str(patch_width))

            stride = self.patch_stride[i]
            if stride == 0:
                max_stride_this_axis = 0
            else:
                max_stride_this_axis = last_valid_coord / stride
            num_strides_this_axis = max_stride_this_axis + 1
            max_strides.append(max_stride_this_axis)
            num_patches *= num_strides_this_axis

        # batch size
        output_shape = [num_patches]

        # topological dimensions
        for dim in self.patch_shape:
            output_shape.append(dim)

        # number of channels
        output_shape.append(X.shape[-1])
        output = numpy.zeros(output_shape, dtype=X.dtype)
        channel_slice = slice(0, X.shape[-1])
        coords = [0] * (num_topological_dimensions + 1)
        keep_going = True
        i = 0

        bboxes = dataset.face_dataset.bboxes

        while keep_going:

            args = [coords[0]]

            for j in xrange(num_topological_dimensions):
                coord = coords[j + 1] * self.patch_stride[j]
                args.append(slice(coord, coord + self.patch_shape[j]))

            args.append(channel_slice)
            patch = X[args]
            output[i, :] = patch
            i += 1

            # increment coordinates
            j = 0
            keep_going = False

            while not keep_going:
                if coords[-(j + 1)] < max_strides[-(j + 1)]:
                    coords[-(j + 1)] += 1
                    keep_going = True
                else:
                    coords[-(j + 1)] = 0
                    if j == num_topological_dimensions:
                        break
                    j = j + 1

        portion = slice(start, stop)

        repeat_times = numpy.ones(X.shape[0], 1)
        if is_face:
            bbox_targets = get_image_bboxes(portion, bboxes)

            outputmaps = convert_bboxes_guided(bbox_targets,
                                               (X.shape[0], X.shape[1]),
                                               self.patch_shape,
                                               area_ratio=self.area_ratio,
                                               stride=stride)
        else:
            targets = numpy.zeros(output_shape[-1])
            outputmaps = repeat_times * targets

        patch_loc = numpy.arange(output_shape[-1])
        patch_locs = repeat_times * patch_loc

        if start is None and stop is None:
            return output, outputmaps.flatten(), patch_locs.flatten()
        elif start is None:
            raise ValueError("You should give a start value.")
        elif start is not None and stop is None:
            return output[start], outputmaps.flatten(
            ), patch_locs[start].flatten()
        else:
            return output[start:stop], outputmaps.flatten(
            ), patch_locs[start:stop].flatten()