def obj_download(bucket_name, dest, key_name, force=False, recursive=False,
multi=False, checksum=False):
bucket = Obj.conn.get_bucket(bucket_name)
if recursive:
logging.info("Starting recursive download %s to %s prefix %s" %
(bucket.name, dest, key_name))
if not os.path.isdir(dest):
logging.error("DEST %s is not a directory." % dest)
return
else:
for key in bucket.list(prefix=key_name):
filename = dest.rstrip(os.sep) + os.sep + key.name
if not check_key_download(bucket, key.name, filename):
continue
logging.info("Downloading key %s (%d) to %s" %
(key, key.size, filename))
if multi and key.size > 5 * 1024 * 1024:
m = MultiPart()
m.start_download(bucket_name, key.name, dest)
else:
download_file(key, filename)
else:
if not key_name:
logging.error("Must specify a key to download or use " +
"recusive option")
return
if os.path.isfile(dest) and not force:
logging.error("File %s already exists " % dest +
"please force flag to overwrite.")
return
else:
key_name = key_name.rstrip(os.path.sep)
key = bucket.get_key(key_name)
if key is None:
logging.error("Key does not exist or if this is a prefix" +
" you need to specify recusive, %s" % key_name)
return
if checksum and not check_key_download(bucket, key.name, dest):
return
# only multipart if we specify and the key is >5MB
if multi and key.size > 5 * 1024 * 1024:
logging.info("Downloading key %s (%d) to %s" %
(key, key.size, dest))
m = MultiPart()
m.start_download(bucket_name, key_name, dest)
else:
key = bucket.get_key(key_name)
download_file(key, dest)
def obj_upload(bucket_name, src, dest_name, recursive=False, multi=False,
checksum=False):
# retranslate to the full absolute path
src = os.path.abspath(src)
# retieve the bucket
bucket = Obj.conn.get_bucket(bucket_name)
try:
size = os.stat(src).st_size
except OSError as e:
logging.error(e)
return
policy = bucket.get_acl()
# use a closure to capture the current bucket
def cancel_multipart_handler(signal, frame):
for upload in bucket.get_all_multipart_uploads():
upload.cancel_upload()
logging.info("Removed incomplete multipart upload: %s" %
str(upload))
umobj_add_handler(signal.SIGINT, cancel_multipart_handler)
if recursive and os.path.isdir(src):
prefix = src.split(os.sep)[-1]
if dest_name:
prefix = dest_name.rstrip(os.sep) + '/' + prefix
for directory in walk_path(prefix):
directory = directory + '/'
if not check_directory(bucket, directory):
create_directory(bucket, directory)
operations = sum([len(files) for r, d, files in
os.walk(src.rstrip(os.sep))])
pbar = progressbar.ProgressBar(maxval=operations)
pbar.start()
count = 0
for root, dirs, files in os.walk(src.rstrip(os.sep)):
# we will not create the base directory
if root != src:
directory = '%s/%s/' % (prefix,
lremove(src, root).lstrip(os.sep))
if not check_directory(bucket, directory):
create_directory(bucket, directory)
count += 1
if count < operations:
pbar.update(count)
for f in files:
filename = root + os.sep + f
try:
file_st = os.stat(filename)
size = file_st.st_size
except OSError as e:
logging.error(e)
continue
if root != src:
keyname = '%s/%s/%s' % (prefix,
lremove(src, root).lstrip(os.sep),
f)
else:
keyname = '%s/%s' % (prefix, f)
if checksum and not check_key_upload(
bucket, keyname, filename):
continue
logging.info("Upload key %s from file %s" %
(keyname, filename))
if (multi and size > 0) or (size > (1024 * 1024 * 1024)):
m = MultiPart()
m.start_upload(bucket_name, keyname, filename, policy)
else:
key = bucket.new_key(keyname)
res = upload_file(key, filename, progress=False)
if res >= 0:
logging.debug("Applying bucket policy %s" % policy)
## set the owner of the policy to the upload user
policy.owner = key.get_acl().owner
key.set_acl(policy)
count += 1
if count < operations:
pbar.update(count)
pbar.finish()
else:
if os.path.isdir(src):
logging.warning("Skipping directory %s, " % src +
"use the recursive option.")
return
if not os.path.isfile(src):
logging.error("File %s does not exist." % src)
return
if dest_name:
current_path = ''
for dir_part in dest_name.lstrip(os.sep).split(os.sep)[:-1]:
current_path = current_path + dir_part + '/'
create_directory(bucket, current_path)
if dest_name.endswith('/'):
key_name = current_path + os.path.basename(src)
else:
key_name = dest_name
else:
key_name = os.path.basename(src)
if checksum and not check_key_upload(bucket, key_name, src):
return
if multi or (size > (1024 * 1024 * 1024)):
logging.info("Starting a multipart upload.")
m = MultiPart()
m.start_upload(bucket_name, key_name, src, policy)
else:
key = bucket.new_key(key_name)
res = upload_file(key, src)
if res >= 0:
logging.debug("Applying bucket policy %s" % policy)
## set the owner of the policy to the upload user
policy.owner = key.get_acl().owner
key.set_acl(policy)
