def publish_images(self,
img_inds,
preproc,
bucket_name,
dummy_upload=False):
ids = [
get_id(str(image_id) + repr(preproc))
for image_id in self.meta['id'][img_inds]
]
source = get_img_source()
if preproc is not None:
raise NotImplementedError
else:
filenames = np.array(self.meta['filename'][img_inds])
conn = boto.connect_s3()
b = conn.create_bucket(bucket_name)
urls = []
i = 0
for image_id, filename in zip(ids, filenames):
i += 1
if i % 100 == 0:
print i / float(len(filenames))
if not dummy_upload:
k = b.new_key(image_id + '.jpg')
k.set_contents_from_file(source.get(str(filename)),
policy='public-read')
urls.append('https://s3.amazonaws.com/' + bucket_name + '/' +
image_id + '.jpg')
return urls
def __init__(self, data=None):
"""
:param data: data specifying how to build this dataset. should uniquely identify dataset among all datasets
:raise: ValueError if instantiated directly
"""
cname = self.__class__.__name__
if cname == 'Imagenet_Base':
print 'The Imagenet base class should not be directly instantiated'
raise ValueError
img_path = self.imagenet_home('images')
self.specific_name = self.__class__.__name__ + '_' + get_id(data)
if not os.path.exists(img_path):
os.makedirs(img_path)
self.img_path = img_path
self.meta_path = self.local_home('meta')
if not os.path.exists(self.meta_path):
os.makedirs(self.meta_path)
self.default_preproc = {
'resize_to': (256, 256),
'mode': 'RGB',
'dtype': 'float32',
'crop': None,
'mask': None,
'normalize': False
}
super(Imagenet_Base, self).__init__(data)
def __init__(self, data=None):
"""
:param data: data specifying how to build this dataset. should uniquely identify dataset among all datasets
:raise: ValueError if instantiated directly
"""
cname = self.__class__.__name__
if cname == "Imagenet_Base":
print "The Imagenet base class should not be directly instantiated"
raise ValueError
img_path = self.imagenet_home("images")
self.specific_name = self.__class__.__name__ + "_" + get_id(data)
if not os.path.exists(img_path):
os.makedirs(img_path)
self.img_path = img_path
self.meta_path = self.local_home("meta")
if not os.path.exists(self.meta_path):
os.makedirs(self.meta_path)
self.default_preproc = {
"resize_to": (256, 256),
"mode": "RGB",
"dtype": "float32",
"crop": None,
"mask": None,
"normalize": False,
}
super(Imagenet_Base, self).__init__(data)
def publish_images(self, img_inds, preproc, bucket_name, dummy_upload=False):
ids = [get_id(str(image_id) + repr(preproc)) for image_id in self.meta["id"][img_inds]]
source = get_img_source()
if preproc is not None:
raise NotImplementedError
else:
filenames = np.array(self.meta["filename"][img_inds])
conn = boto.connect_s3()
b = conn.create_bucket(bucket_name)
urls = []
i = 0
for image_id, filename in zip(ids, filenames):
i += 1
if i % 100 == 0:
print i / float(len(filenames))
if not dummy_upload:
k = b.new_key(image_id + ".jpg")
k.set_contents_from_file(source.get(str(filename)), policy="public-read")
urls.append("https://s3.amazonaws.com/" + bucket_name + "/" + image_id + ".jpg")
return urls
