def upload_large(self, source_path):
source_size = os.stat(source_path).st_size
source_key = md5(str(uuid.uuid4())).hexdigest() + ".mp4"
print "source path %s" % source_path
print "source key %s" % source_key
mp = self.bucket.initiate_multipart_upload(source_key)
chunk_size = 52428800
chunk_count = int(math.ceil(source_size / float(chunk_size)))
for i in range(chunk_count):
offset = chunk_size * i
bytes = min(chunk_size, source_size - offset)
with FileChunkIO(source_path, 'r', offset=offset,
bytes=bytes) as fp:
mp.upload_part_from_file(fp, part_num=i + 1)
mp.complete_upload()
k = Key(self.bucket)
k.key = source_key
k.set_remote_metadata(
{
'iname':
os.path.basename(source_path),
"iscale":
str(source_size),
"itime":
time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(int(time.time())))
}, {}, True)
remote_key = self.bucket.get_key(source_key)
print "Key name", len(remote_key.metadata)
def upload(self, filename, filepath):
k = Key(self.fusion_bucket)
k.key = "%s/%s/%s" % (self.folder, get_current_date(), filename)
k.set_contents_from_filename(filepath, policy='public-read')
content_type = mimetypes.guess_type(filename)[0]
k.set_remote_metadata({'Cache-Control': self.cache_header, 'Content-Type': content_type}, {}, True)
return k.generate_url(expires_in=0, query_auth=False)
def upload(self, filename, filepath):
k = Key(self.fusion_bucket)
k.key = "%s/%s/%s" % (self.folder, get_current_date(), filename)
k.set_contents_from_filename(filepath, policy='public-read')
content_type = mimetypes.guess_type(filename)[0]
k.set_remote_metadata(
{
'Cache-Control': self.cache_header,
'Content-Type': content_type
}, {}, True)
return k.generate_url(expires_in=0, query_auth=False)
def store(self,
in_bucket=None,
key_name=None,
metadata=None,
quality=95,
public=True):
"""Store the loaded image into the given bucket with the given key name. Tag
it with metadata if provided. Make the Image public and return its url"""
if not in_bucket:
raise InvalidParameterException("No in_bucket specified")
if not key_name:
raise InvalidParameterException("No key_name specified")
if not self.image:
raise RTFMException(
"No image loaded! You must call fetch() before store()")
if metadata:
if type(metadata) is not dict:
raise RTFMException("metadata must be a dict")
else:
metadata = {}
metadata['Content-Type'] = 'image/jpeg'
log.info("Storing image into bucket %s/%s" % (in_bucket, key_name))
# Export image to a string
sio = BytesIO()
self.image.save(sio, 'JPEG', quality=quality, optimize=True)
contents = sio.getvalue()
sio.close()
# Get the bucket
bucket = self.s3_conn.get_bucket(in_bucket)
# Create a key containing the image. Make it public
k = Key(bucket)
k.key = key_name
k.set_contents_from_string(contents)
k.set_remote_metadata(metadata, {}, True)
if public:
k.set_acl('public-read')
# Return the key's url
return k.generate_url(method='GET',
expires_in=0,
query_auth=False,
force_http=False)
def upload_asset_to_s3(asset, access_key, secret_key, bucket):
connection = S3Connection(access_key, secret_key)
asset_bucket = connection.get_bucket(bucket)
asset_path = asset.asset_file.path
asset_file = open(asset_path)
tag_string = "-".join([tag.name.replace("/", "-") for tag in asset.app.tags.all()])
product = asset.app.product.name
file_extension = os.path.splitext(asset_path)[-1]
version = asset.app.version
key_name = "%s-%s-%s%s" % (product, tag_string, version, file_extension)
app_key = Key(asset_bucket, key_name)
app_key.set_contents_from_file(open(asset_path))
app_key.make_public()
metadata = {'version' : version, 'product' : product}
app_key.set_remote_metadata(metadata, {}, preserve_acl=True)
asset.external_url = app_key.generate_url(0, query_auth=False)
asset.save()
def upload_asset_to_s3(asset, access_key, secret_key, bucket):
connection = S3Connection(access_key, secret_key)
asset_bucket = connection.get_bucket(bucket)
asset_path = asset.asset_file.path
asset_file = open(asset_path)
tag_string = "-".join(
[tag.name.replace("/", "-") for tag in asset.app.tags.all()])
product = asset.app.product.name
file_extension = os.path.splitext(asset_path)[-1]
version = asset.app.version
key_name = "%s-%s-%s%s" % (product, tag_string, version, file_extension)
app_key = Key(asset_bucket, key_name)
app_key.set_contents_from_file(open(asset_path))
app_key.make_public()
metadata = {'version': version, 'product': product}
app_key.set_remote_metadata(metadata, {}, preserve_acl=True)
asset.external_url = app_key.generate_url(0, query_auth=False)
asset.save()
print conn.get_all_buckets()
for b in conn.get_all_buckets():
print b
#bucket.set_acl('public-read')
###
from boto.s3.key import Key
k = Key(bucket)
k.key = 'test1.dat'
k.set_contents_from_string('hello123')
#k.set_acl('public-read')
k.set_remote_metadata( {'x-scal-test': { 'key1':'value1' }}, [], True )
c = Key(bucket)
c.key = 'test1.dat'
print c.get_contents_as_string()
print c.get_acl()
print c.metadata
###
###
full_bucket = conn.get_bucket('test-bucket')
# It's full of keys. Delete them all.
for key in full_bucket.list():
key.delete()
