os.makedirs(FLAGS.feature_dir)
except OSError:
pass
def read_image(self, name):
"""Reads the image and does the manipulation
"""
img = datasets.imread_rgb(name)
return datasets.manipulate(img, None, None, None, CENTER_CROP)
def map(self, key, value):
"""key will be dummy, and value will be the image filename
"""
imagename = os.path.basename(value)
feature = self._conv.process( \
self.read_image(os.path.join(FLAGS.image_dir, value)),\
convbuffer = self._buffer)
np.save(os.path.join(FLAGS.feature_dir, imagename), feature)
yield self._hostname, imagename
mapreducer.REGISTER_DEFAULT_MAPPER(FeatureExtractionMapper)
# for Reduce, we will simply use the identity reducer.
mapreducer.REGISTER_DEFAULT_REDUCER(mapreducer.IdentityReducer)
mapreducer.REGISTER_DEFAULT_READER(mapreducer.FileReader)
mapreducer.REGISTER_DEFAULT_WRITER(mapreducer.PickleWriter)
if __name__ == "__main__":
launcher.launch()
mean = sum(v[0] for v in values) / sum(v[1] for v in values)
return mean
class StdMapper(mapreducer.BasicMapper):
def set_up(self):
self._m = pickle.load(open(FLAGS.mean))['m']
def map(self, key, value):
data = np.load(FLAGS.feature_file % key)
data -= self._m
yield "std", ((data**2).sum(0), data.shape[0])
class StdReducer(mapreducer.BasicReducer):
def reduce(self, key, values):
std = sum(v[0] for v in values) / sum(v[1] for v in values)
std = np.sqrt(std + FLAGS.reg)
return std
mapreducer.REGISTER_MAPPER(MeanMapper)
mapreducer.REGISTER_REDUCER(MeanReducer)
mapreducer.REGISTER_MAPPER(StdMapper)
mapreducer.REGISTER_REDUCER(StdReducer)
mapreducer.REGISTER_DEFAULT_READER(mapreducer.IterateReader)
mapreducer.REGISTER_DEFAULT_WRITER(mapreducer.PickleWriter)
if __name__ == "__main__":
launcher.launch()
