def start_upload(self, bucketname, keyname, stream, policy, threads=4): self.bucketname = bucketname self.stream = stream headers = {} obj = self.connect() bucket = obj.get_bucket(self.bucketname) mtype = mimetypes.guess_type(keyname)[0] or 'application/octet-stream' headers.update({'Content-Type': mtype}) mp = bucket.initiate_multipart_upload(keyname, headers=headers) def cancel_upload_handler(signal, frame): self.cancel_upload(mp) umobj_add_handler(signal.SIGINT, cancel_upload_handler) self.mp_id = mp.id bytes_per_chunk = (1024*1024*10) logging.info("Chunk Size: %16d " % bytes_per_chunk) # read initial bytes from data stream and upload in parts bytes_in = stream.read(bytes_per_chunk) part_num = 1 logging.info("Starting multipart uploads from stream") try: while(bytes_in): # bytes are read from stream as a string, wrap in StringIO # object to be able to use upload_part_from_file function mp.upload_part_from_file(StringIO(bytes_in), part_num=part_num) part_num += 1 bytes_in = stream.read(bytes_per_chunk) except Exception as e: log.error(e) self.cancel_upload(mp) sys.exit(1) if not stream.read(1): mp.complete_upload() key = bucket.get_key(keyname) logging.debug("%s : Applying bucket policy %s" % (mp.id, policy)) policy.owner = key.get_acl().owner key.set_acl(policy) else: self.cancel_upload(mp)
def obj_upload(bucket_name, src, dest_name, recursive=False, multi=False, checksum=False): # retranslate to the full absolute path src = os.path.abspath(src) # retieve the bucket bucket = Obj.conn.get_bucket(bucket_name) try: size = os.stat(src).st_size except OSError as e: logging.error(e) return policy = bucket.get_acl() # use a closure to capture the current bucket def cancel_multipart_handler(signal, frame): for upload in bucket.get_all_multipart_uploads(): upload.cancel_upload() logging.info("Removed incomplete multipart upload: %s" % str(upload)) umobj_add_handler(signal.SIGINT, cancel_multipart_handler) if recursive and os.path.isdir(src): prefix = src.split(os.sep)[-1] if dest_name: prefix = dest_name.rstrip(os.sep) + '/' + prefix for directory in walk_path(prefix): directory = directory + '/' if not check_directory(bucket, directory): create_directory(bucket, directory) operations = sum([len(files) for r, d, files in os.walk(src.rstrip(os.sep))]) pbar = progressbar.ProgressBar(maxval=operations) pbar.start() count = 0 for root, dirs, files in os.walk(src.rstrip(os.sep)): # we will not create the base directory if root != src: directory = '%s/%s/' % (prefix, lremove(src, root).lstrip(os.sep)) if not check_directory(bucket, directory): create_directory(bucket, directory) count += 1 if count < operations: pbar.update(count) for f in files: filename = root + os.sep + f try: file_st = os.stat(filename) size = file_st.st_size except OSError as e: logging.error(e) continue if root != src: keyname = '%s/%s/%s' % (prefix, lremove(src, root).lstrip(os.sep), f) else: keyname = '%s/%s' % (prefix, f) if checksum and not check_key_upload( bucket, keyname, filename): continue logging.info("Upload key %s from file %s" % (keyname, filename)) if (multi and size > 0) or (size > (1024 * 1024 * 1024)): m = MultiPart() m.start_upload(bucket_name, keyname, filename, policy) else: key = bucket.new_key(keyname) res = upload_file(key, filename, progress=False) if res >= 0: logging.debug("Applying bucket policy %s" % policy) ## set the owner of the policy to the upload user policy.owner = key.get_acl().owner key.set_acl(policy) count += 1 if count < operations: pbar.update(count) pbar.finish() else: if os.path.isdir(src): logging.warning("Skipping directory %s, " % src + "use the recursive option.") return if not os.path.isfile(src): logging.error("File %s does not exist." % src) return if dest_name: current_path = '' for dir_part in dest_name.lstrip(os.sep).split(os.sep)[:-1]: current_path = current_path + dir_part + '/' create_directory(bucket, current_path) if dest_name.endswith('/'): key_name = current_path + os.path.basename(src) else: key_name = dest_name else: key_name = os.path.basename(src) if checksum and not check_key_upload(bucket, key_name, src): return if multi or (size > (1024 * 1024 * 1024)): logging.info("Starting a multipart upload.") m = MultiPart() m.start_upload(bucket_name, key_name, src, policy) else: key = bucket.new_key(key_name) res = upload_file(key, src) if res >= 0: logging.debug("Applying bucket policy %s" % policy) ## set the owner of the policy to the upload user policy.owner = key.get_acl().owner key.set_acl(policy)