os.makedirs(FLAGS.feature_dir) except OSError: pass def read_image(self, name): """Reads the image and does the manipulation """ img = datasets.imread_rgb(name) return datasets.manipulate(img, None, None, None, CENTER_CROP) def map(self, key, value): """key will be dummy, and value will be the image filename """ imagename = os.path.basename(value) feature = self._conv.process( \ self.read_image(os.path.join(FLAGS.image_dir, value)),\ convbuffer = self._buffer) np.save(os.path.join(FLAGS.feature_dir, imagename), feature) yield self._hostname, imagename mapreducer.REGISTER_DEFAULT_MAPPER(FeatureExtractionMapper) # for Reduce, we will simply use the identity reducer. mapreducer.REGISTER_DEFAULT_REDUCER(mapreducer.IdentityReducer) mapreducer.REGISTER_DEFAULT_READER(mapreducer.FileReader) mapreducer.REGISTER_DEFAULT_WRITER(mapreducer.PickleWriter) if __name__ == "__main__": launcher.launch()
mean = sum(v[0] for v in values) / sum(v[1] for v in values) return mean class StdMapper(mapreducer.BasicMapper): def set_up(self): self._m = pickle.load(open(FLAGS.mean))['m'] def map(self, key, value): data = np.load(FLAGS.feature_file % key) data -= self._m yield "std", ((data**2).sum(0), data.shape[0]) class StdReducer(mapreducer.BasicReducer): def reduce(self, key, values): std = sum(v[0] for v in values) / sum(v[1] for v in values) std = np.sqrt(std + FLAGS.reg) return std mapreducer.REGISTER_MAPPER(MeanMapper) mapreducer.REGISTER_REDUCER(MeanReducer) mapreducer.REGISTER_MAPPER(StdMapper) mapreducer.REGISTER_REDUCER(StdReducer) mapreducer.REGISTER_DEFAULT_READER(mapreducer.IterateReader) mapreducer.REGISTER_DEFAULT_WRITER(mapreducer.PickleWriter) if __name__ == "__main__": launcher.launch()