class OssStorage(Storage): """ Aliyun OSS Storage """ def __init__(self, access_key_id=None, access_key_secret=None, end_point=None, bucket_name=None): self.access_key_id = access_key_id if access_key_id else _get_config( 'OSS_ACCESS_KEY_ID') self.access_key_secret = access_key_secret if access_key_secret \ else _get_config( 'OSS_ACCESS_KEY_SECRET') self.end_point = _normalize_endpoint( end_point if end_point else _get_config('OSS_ENDPOINT')) self.bucket_name = bucket_name if bucket_name else _get_config( 'OSS_BUCKET_NAME') self.auth = Auth(self.access_key_id, self.access_key_secret) self.service = Service(self.auth, self.end_point) self.bucket = Bucket(self.auth, self.end_point, self.bucket_name) # try to get bucket acl to check bucket exist or not # try: # self.bucket.get_bucket_acl().acl # except oss2.exceptions.NoSuchBucket: # raise SuspiciousOperation( # "Bucket '%s' does not exist." % self.bucket_name) def _get_key_name(self, name): """ Get the object key name in OSS, e.g., location: /media/ input : test.txt output : media/test.txt """ base_path = force_str(self.location) final_path = urljoin(base_path + "/", name) name = os.path.normpath(final_path.lstrip('/')) if six.PY2: name = name.encode('utf-8') return name def _open(self, name, mode='rb'): logger().debug("name: %s, mode: %s", name, mode) if mode != "rb": raise ValueError("OSS files can only be opened in read-only mode") target_name = self._get_key_name(name) logger().debug("target name: %s", target_name) try: # Load the key into a temporary file tmpf = SpooledTemporaryFile(max_size=10 * 1024 * 1024) # 10MB obj = self.bucket.get_object(target_name) logger().info("content length: %d, requestid: %s", obj.content_length, obj.request_id) if obj.content_length is None: shutil.copyfileobj(obj, tmpf) else: oss2.utils.copyfileobj_and_verify(obj, tmpf, obj.content_length, request_id=obj.request_id) tmpf.seek(0) return OssFile(tmpf, target_name, self) except oss2.exceptions.NoSuchKey: raise OssError("%s does not exist" % name) except Exception: raise OssError("Failed to open %s" % name) def _save(self, name, content): target_name = self._get_key_name(name) logger().debug("target name: %s", target_name) logger().debug("content: %s", content) self.bucket.put_object(target_name, content) return os.path.normpath(name) def create_dir(self, dirname): target_name = self._get_key_name(dirname) if not target_name.endswith('/'): target_name += '/' self.bucket.put_object(target_name, '') def exists(self, name): target_name = self._get_key_name(name) logger().debug("name: %s, target name: %s", name, target_name) if name.endswith("/"): result = self.bucket.list_objects(prefix=target_name, delimiter='', marker='', max_keys=1) if len(result.object_list) == 0: logger().debug("object list: %s", result.object_list) else: logger().debug("object list: %s", result.object_list[0].key) return bool(result.object_list) exist = self.bucket.object_exists(target_name) logger().debug("'%s' exist: %s", target_name, exist) if not exist: name2 = name + "/" logger().debug("to check %s", name2) return self.exists(name2) return exist def get_file_meta(self, name): name = self._get_key_name(name) return self.bucket.get_object_meta(name) def size(self, name): file_meta = self.get_file_meta(name) return file_meta.content_length def modified_time(self, name): file_meta = self.get_file_meta(name) return datetime.fromtimestamp(file_meta.last_modified) created_time = accessed_time = modified_time def get_modified_time(self, name): file_meta = self.get_file_meta(name) if settings.USE_TZ: return datetime.utcfromtimestamp( file_meta.last_modified).replace(tzinfo=utc) else: return datetime.fromtimestamp(file_meta.last_modified) get_created_time = get_accessed_time = get_modified_time def content_type(self, name): name = self._get_key_name(name) file_info = self.bucket.head_object(name) return file_info.content_type def listdir(self, name): if name == ".": name = "" name = self._get_key_name(name) if not name.endswith('/'): name += "/" logger().debug("name: %s", name) files = [] dirs = [] for obj in ObjectIterator(self.bucket, prefix=name, delimiter='/'): if obj.is_prefix(): dirs.append(obj.key) else: files.append(obj.key) logger().debug("dirs: %s", list(dirs)) logger().debug("files: %s", files) return dirs, files def url(self, name, expire=24 * 60 * 60): key = self._get_key_name(name) return self.bucket.sign_url('GET', key, expire) def delete(self, name): name = self._get_key_name(name) logger().debug("delete name: %s", name) return self.bucket.delete_object(name) def delete_with_slash(self, dirname): name = self._get_key_name(dirname) if not name.endswith('/'): name += '/' logger().debug("delete name: %s", name) return self.bucket.delete_object(name)
class OssStorage(Storage): """ Aliyun OSS Storage """ def __init__(self, access_key_id=None, access_key_secret=None, end_point=None, bucket_name=None, expire_time=None): self.access_key_id = access_key_id if access_key_id else _get_config( 'OSS_ACCESS_KEY_ID') self.access_key_secret = access_key_secret if access_key_secret else _get_config( 'OSS_ACCESS_KEY_SECRET') self.end_point = _normalize_endpoint( end_point if end_point else _get_config('OSS_ENDPOINT')) self.bucket_name = bucket_name if bucket_name else _get_config( 'OSS_BUCKET_NAME') self.expire_time = expire_time if expire_time else int( _get_config('OSS_EXPIRE_TIME', default=60 * 60 * 24 * 30)) self.auth = Auth(self.access_key_id, self.access_key_secret) self.service = Service(self.auth, self.end_point) self.bucket = Bucket(self.auth, self.end_point, self.bucket_name) # try to get bucket acl to check bucket exist or not try: self.bucket_acl = self.bucket.get_bucket_acl().acl except oss2.exceptions.NoSuchBucket: raise SuspiciousOperation("Bucket '%s' does not exist." % self.bucket_name) def _get_key_name(self, name): """ Get the object key name in OSS, e.g., location: /media/ input : test.txt output : media/test.txt """ # urljoin won't work if name is absolute path name = name.lstrip('/') base_path = force_text(self.location) final_path = urljoin(base_path + "/", name) name = os.path.normpath(final_path.lstrip('/')) # Add / to the end of path since os.path.normpath will remove it if final_path.endswith('/') and not name.endswith('/'): name += '/' if six.PY2: name = name.encode('utf-8') # Store filenames with forward slashes, even on Windows. return name.replace('\\', '/') def _open(self, name, mode='rb'): logger().debug("name: %s, mode: %s", name, mode) if mode != "rb": raise ValueError("OSS files can only be opened in read-only mode") target_name = self._get_key_name(name) logger().debug("target name: %s", target_name) try: # Load the key into a temporary file tmpf = SpooledTemporaryFile(max_size=10 * 1024 * 1024) # 10MB obj = self.bucket.get_object(target_name) logger().info("content length: %d, requestid: %s", obj.content_length, obj.request_id) if obj.content_length is None: shutil.copyfileobj(obj, tmpf) else: oss2.utils.copyfileobj_and_verify(obj, tmpf, obj.content_length, request_id=obj.request_id) tmpf.seek(0) return OssFile(tmpf, target_name, self) except oss2.exceptions.NoSuchKey: raise OssError("%s does not exist" % name) except: raise OssError("Failed to open %s" % name) def _save(self, name, content): target_name = self._get_key_name(name) logger().debug("target name: %s", target_name) logger().debug("content: %s", content) self.bucket.put_object(target_name, content) return os.path.normpath(name) def create_dir(self, dirname): target_name = self._get_key_name(dirname) if not target_name.endswith('/'): target_name += '/' self.bucket.put_object(target_name, '') def exists(self, name): target_name = self._get_key_name(name) logger().debug("name: %s, target name: %s", name, target_name) if name.endswith("/"): # This looks like a directory, but OSS has no concept of directories # need to check whether the key starts with this prefix result = self.bucket.list_objects(prefix=target_name, delimiter='', marker='', max_keys=1) if len(result.object_list) == 0: logger().debug("object list: %s", result.object_list) else: logger().debug("object list: %s", result.object_list[0].key) return bool(result.object_list) exist = self.bucket.object_exists(target_name) logger().debug("'%s' exist: %s", target_name, exist) if not exist: # It's not a file, but it might be a directory. Check again that it's not a directory. name2 = name + "/" logger().debug("to check %s", name2) return self.exists(name2) return exist def get_file_meta(self, name): name = self._get_key_name(name) return self.bucket.get_object_meta(name) def size(self, name): file_meta = self.get_file_meta(name) return file_meta.content_length def modified_time(self, name): file_meta = self.get_file_meta(name) return datetime.fromtimestamp(file_meta.last_modified) created_time = accessed_time = modified_time def get_modified_time(self, name): file_meta = self.get_file_meta(name) if settings.USE_TZ: return datetime.utcfromtimestamp( file_meta.last_modified).replace(tzinfo=utc) else: return datetime.fromtimestamp(file_meta.last_modified) get_created_time = get_accessed_time = get_modified_time def content_type(self, name): name = self._get_key_name(name) file_info = self.bucket.head_object(name) return file_info.content_type def listdir(self, name): if name == ".": name = "" name = self._get_key_name(name) if not name.endswith('/'): name += "/" logger().debug("name: %s", name) files = [] dirs = [] for obj in ObjectIterator(self.bucket, prefix=name, delimiter='/'): if obj.is_prefix(): dirs.append(obj.key) else: files.append(obj.key) logger().debug("dirs: %s", list(dirs)) logger().debug("files: %s", files) return dirs, files def url(self, name): key = self._get_key_name(name) str = self.bucket.sign_url('GET', key, expires=self.expire_time) if self.bucket_acl != BUCKET_ACL_PRIVATE: idx = str.find('?') if idx > 0: str = str[:idx].replace('%2F', '/') return str def delete(self, name): name = self._get_key_name(name) logger().debug("delete name: %s", name) result = self.bucket.delete_object(name) def delete_with_slash(self, dirname): name = self._get_key_name(dirname) if not name.endswith('/'): name += '/' logger().debug("delete name: %s", name) result = self.bucket.delete_object(name)
class OssStorage(Storage): """ Aliyun OSS Storage """ def __init__(self, access_key_id=None, access_key_secret=None, end_point=None, bucket_name=None): self.access_key_id = access_key_id if access_key_id else _get_config( 'OSS_ACCESS_KEY_ID') self.access_key_secret = access_key_secret if access_key_secret else _get_config( 'OSS_ACCESS_KEY_SECRET') self.end_point = _normalize_endpoint( end_point if end_point else _get_config('OSS_ENDPOINT')) self.bucket_name = bucket_name if bucket_name else _get_config( 'OSS_BUCKET_NAME') sts_token = getattr(settings, 'ALIYUN_STS_TOKEN', None) # 这里表示如果有sts_token,需要使用stsauth进行鉴权 if sts_token: self.auth = StsAuth(self.access_key_id, self.access_key_secret, sts_token) else: self.auth = Auth(self.access_key_id, self.access_key_secret) self.service = Service(self.auth, self.end_point) use_oss_internal = getattr(settings, 'OSS_USE_INTERNAL', None) # 这里表示,如果是阿里云的内网机器,默认走内网的end_point,否则使用外网的end_point # 使用内网end_point,速度快,不收费 if use_oss_internal: self.end_point_internal = _normalize_endpoint( end_point if end_point else _get_config('OSS_ENDPOINT_INTERNAL' )) self.bucket = Bucket(self.auth, self.end_point_internal, self.bucket_name) else: self.bucket = Bucket(self.auth, self.end_point, self.bucket_name) self.bucket_public = Bucket(self.auth, self.end_point, self.bucket_name) # try to get bucket acl to check bucket exist or not try: self.bucket.get_bucket_acl().acl except oss2.exceptions.NoSuchBucket: raise SuspiciousOperation("Bucket '%s' does not exist." % self.bucket_name) def _get_key_name(self, name): """ Get the object key name in OSS, e.g., location: /media/ input : test.txt output : media/test.txt """ base_path = force_text(self.location) final_path = urljoin(base_path + "/", name) name = os.path.normpath(final_path.lstrip('/')) name = name.replace('\\', '/') if six.PY2: name = name.encode('utf-8') return name def _open(self, name, mode='rb'): logger().debug("name: %s, mode: %s", name, mode) if mode != "rb": raise ValueError("OSS files can only be opened in read-only mode") target_name = self._get_key_name(name) logger().debug("target name: %s", target_name) try: # Load the key into a temporary file tmpf = NamedTemporaryFile() # 10MB obj = self.bucket.get_object(target_name) logger().info("content length: %d, requestid: %s", obj.content_length, obj.request_id) if obj.content_length is None: shutil.copyfileobj(obj, tmpf) else: oss2.utils.copyfileobj_and_verify(obj, tmpf, obj.content_length, request_id=obj.request_id) tmpf.seek(0) return OssFile(tmpf, target_name, self) except oss2.exceptions.NoSuchKey: raise OssError("%s does not exist" % name) except: raise OssError("Failed to open %s" % name) def _save(self, name, content): target_name = self._get_key_name(name) logger().debug("target name: %s", target_name) logger().debug("content: %s", content) self.bucket.put_object(target_name, content) return os.path.normpath(name) def create_dir(self, dirname): target_name = self._get_key_name(dirname) if not target_name.endswith('/'): target_name += '/' self.bucket.put_object(target_name, '') def exists(self, name): target_name = self._get_key_name(name) logger().debug("name: %s, target name: %s", name, target_name) if name.endswith("/"): # This looks like a directory, but OSS has no concept of directories # need to check whether the key starts with this prefix result = self.bucket.list_objects(prefix=target_name, delimiter='', marker='', max_keys=1) if len(result.object_list) == 0: logger().debug("object list: %s", result.object_list) else: logger().debug("object list: %s", result.object_list[0].key) return bool(result.object_list) exist = self.bucket.object_exists(target_name) logger().debug("'%s' exist: %s", target_name, exist) if not exist: # It's not a file, but it might be a directory. Check again that it's not a directory. name2 = name + "/" logger().debug("to check %s", name2) return self.exists(name2) return exist def get_file_meta(self, name): name = self._get_key_name(name) return self.bucket.get_object_meta(name) def size(self, name): file_meta = self.get_file_meta(name) return file_meta.content_length def modified_time(self, name): file_meta = self.get_file_meta(name) return datetime.fromtimestamp(file_meta.last_modified) created_time = accessed_time = modified_time def get_modified_time(self, name): file_meta = self.get_file_meta(name) if settings.USE_TZ: return datetime.utcfromtimestamp( file_meta.last_modified).replace(tzinfo=utc) else: return datetime.fromtimestamp(file_meta.last_modified) get_created_time = get_accessed_time = get_modified_time def content_type(self, name): name = self._get_key_name(name) file_info = self.bucket.head_object(name) return file_info.content_type def listdir(self, name): if name == ".": name = "" name = self._get_key_name(name) if not name.endswith('/'): name += "/" logger().debug("name: %s", name) files = [] dirs = [] for obj in ObjectIterator(self.bucket, prefix=name, delimiter='/'): if obj.is_prefix(): dirs.append(obj.key) else: files.append(obj.key) logger().debug("dirs: %s", list(dirs)) logger().debug("files: %s", files) return dirs, files def url(self, name, expire=60 * 60): key = self._get_key_name(name) custom_domain = getattr(settings, 'OSS_CUSTOM_DOMAIN', None) # return self.bucket.sign_url('GET', key, expire) # 这里一般是提供给浏览器用的,所以走外网的end_point url = self.bucket_public.sign_url('GET', key, expire) url = url.replace('%2F', '/') if custom_domain: url_list = list(urlsplit(url)) custom_domain_list = list(urlsplit(custom_domain)) url_list[0] = custom_domain_list[0] url_list[1] = custom_domain_list[1] url = urlunsplit(url_list) return url def delete(self, name): name = self._get_key_name(name) logger().debug("delete name: %s", name) result = self.bucket.delete_object(name) def delete_with_slash(self, dirname): name = self._get_key_name(dirname) if not name.endswith('/'): name += '/' logger().debug("delete name: %s", name) result = self.bucket.delete_object(name)