def private_download_url(self, bucket, key, expires=config.get_default('expires'), header=None, internal=False): """ 从私有空间下载文件的url @param bucket: string类型, 空间名称 @param key: string类型,下载数据在空间中的名称 @param expires: integer类型, 下载链接有效时间,单位为秒 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return string, 从私有空间下载文件和数据的url """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') if expires is not None: expires += int(time.time()) header['Expires'] = s(str(expires)) signature = self.signature(bucket, key, 'get', header) query = { 'UCloudPublicKey': self._public_key(), 'Expires': str(expires), 'Signature': signature } query_str = urllib.urlencode(query) if internal: return 'http://{0}{1}/{2}?{3}'.format(bucket, config.get_default('download_suffix'), key, query_str) else: return 'http://{0}{1}/{2}?UCloudPublicKey={3}&Expires={4}&Signature={5}'.format(bucket, config.get_default('download_suffix'), key, self._public_key(), str(expires), signature)
def download_file(self, bucket, key, localfile, isprivate=True, expires=config.get_default('expires'), content_range=None, header=None): """ 下载UFile文件并且保存为本地文件 @param bucket: string类型, UFile空间名称 @param key: string类型, 下载文件在空间中的名称 @param localfile: string类型,要保存的本地文件名称 @param isprivate: boolean类型,如果为私有空间则为True @param expires: integer类型,私有文件链接有效时间 @param content_range: tuple类型,元素为两个整型 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') if isinstance(content_range, tuple) and len(content_range) == 2: header['Range'] = 'bytes=' + '-'.join(map(lambda x: str(x), content_range)) if not isprivate: url = self.public_download_url(bucket, key) else: url = self.private_download_url(bucket, key, expires, header, True) logger.info('get ufile url:{0}'.format(url)) return _download_file(url, header, localfile)
def list(self, bucket, marker, limit, header=None): """ @param bucket: string类型,上传空间名称 @param marker: string类型,上一次查找返回的位置信息,本次列表起始查找的文件名,默认为空 @param limit: integer类型, 每次列表返回的最大文件个数,服务端限制为1000,最新限制值请参考官方文档 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') mime_type = s(Mimetype.from_file(localfile)) file_size = os.path.getsize(localfile) header['Content-Type'] = mime_type authorization = self.authorization('put', bucket, key, header) header['Authorization'] = authorization header['Content-Length'] = str(file_size) url = ufile_put_url(bucket, key) logger.info('start put file {0} to bucket {1} as {2}'.format(localfile, bucket, key)) logger.info('put UFile url: {0}'.format(url)) logger.info('request header:\n{0}'.format(json.dumps(header, indent=4))) return _put_file(url, header, localfile)
def describebucket(self, bucket=None, offset=0, limit=10, header=None): """ 获取空间的信息,如果不提供空间名称,则获取所有空间的信息 @param bucketname: string类型, 空间名称 @param offset: integer类型, 起始空间编码,当提供空间名时无效 @param limit: integer类型,获取空间数量,当提供具体空间名时无效 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') param = dict() param['Action'] = 'DescribeBucket' if bucket is not None: param['BucketName'] = bucket param['Offset'] = s(str(offset)) param['Limit'] = s(str(limit)) signature = self.__auth.bucket_signature(param) param['Signature'] = signature logger.info('start request the bucket {0} details'.format(bucket)) return _bucket_request(UCLOUD_API_URL, param, header)
def createbucket(self, bucket, buckettype='private', domainlist=None, header=None): """ 创建新的空间 @param bucket: string类型,空间名称 @param buckettype: string类型,'private' 或者 'public' @param domainlist: list 类型, 要绑定的域名列表 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') param = dict() param['Action'] = 'CreateBucket' param['BucketName'] = bucket param['Type'] = buckettype if domainlist is None: domainlist = [] for number, item in enumerate(domainlist): param['Domain.{0}'.format(number)] = item signature = self.__auth.bucket_signature(param) param['Signature'] = signature logger.info('start create bucket {0}'.format(bucket)) return _bucket_request(UCLOUD_API_URL, param, header)
def getfilelist(self, bucket, offset=0, limit=20, header=None): """ 获取空间中文件列表 @param bucket: string类型,空间名称 @param offset: integer类型,文件列表偏移位置 @param limit: integer类型,返回文件数量 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') param = dict() param['Action'] = 'GetFileList' param['BucketName'] = bucket param['Offset'] = s(str(offset)) param['Limit'] = s(str(limit)) signature = self.__auth.bucket_signature(param) param['Signature'] = signature logger.info('start request the file list of bucket {0}'.format(bucket)) return _bucket_request(UCLOUD_API_URL, param, header)
def __finishupload(self): """ 完成分片上传的请求 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if self.__header is None: self.__header = dict() else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') self.__header['Content-Type'] = 'text/plain' logger.info(self.etaglist) data = ','.join(self.etaglist) logger.info(data) self.__header['Content-Length'] = str(len(data)) authorization = self.authorization('post', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization logger.info(json.dumps(self.__header, indent=4)) params = {'uploadId': self.uploadid} url = finishsharding_url(self.__bucket, self.__key) logger.info('start finish sharding request') return _finishsharding(url, params, self.__header, data)
def updatebucket(self, bucket, buckettype, header=None): """ 更新空间的属性 @param bucket: string类型,空间名称 @param buckettype: string类型, 'private' 或者 'string' @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') param = dict() param['Action'] = 'UpdateBucket' param['BucketName'] = bucket param['Type'] = buckettype signature = self.__auth.bucket_signature(param) param['signature'] = signature return _bucket_request(UCLOUD_API_URL, param, header)
def __initialsharding(self): """ 初始化分片请求 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if self.__header is None: self.__header = {} else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') self.__header['Content-Length'] = str(0) self.__header['Content-Type'] = 'text/plain' authorization = self.authorization('post', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization url = initialsharding_url(self.__bucket, self.__key) logger.info('start initialize sharding') logger.info('initial sharding url: {0}'.format(url)) return _initialsharding(url, self.__header)
def __finishupload(self): """ 完成分片上传的请求 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if self.__header is None: self.__header = dict() else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') self.__header['Content-Type'] = 'text/plain' logger.info(self.etaglist) data = ','.join(self.etaglist) logger.info(data) self.__header['Content-Length'] = len(data) authorization = self.authorization('post', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization logger.info(json.dumps(self.__header, indent=4)) params = {'uploadId': self.uploadid} url = finishsharding_url(self.__bucket, self.__key) logger.info('start finish sharding request') return _finishsharding(url, params, self.__header, data)
def __initialsharding(self): """ 初始化分片请求 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if self.__header is None: self.__header = {} else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') self.__header['Content-Length'] = 0 self.__header['Content-Type'] = 'text/plain' authorization = self.authorization('post', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization url = initialsharding_url(self.__bucket, self.__key) logger.info('start initialize sharding') logger.info('initial sharding url: {0}'.format(url)) return _initialsharding(url, self.__header)
def putfile(self, bucket, key, localfile, header=None): """ upload localfile to bucket as key @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param localfile: string类型,本地文件名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') mime_type = s(Mimetype.from_file(localfile)) file_size = os.path.getsize(localfile) header['Content-Type'] = mime_type authorization = self.authorization('put', bucket, key, header) header['Authorization'] = authorization header['Content-Length'] = file_size url = ufile_put_url(bucket, key) logger.info('start put file {0} to bucket {1} as {2}'.format( localfile, bucket, key)) logger.info('put UFile url: {0}'.format(url)) logger.info('request header:\n{0}'.format(json.dumps(header, indent=4))) return _put_file(url, header, localfile)
def putstream(self, bucket, key, stream, mime_type=None, header=None): """ 上传二进制流到空间 @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param stream: 二进制数据流,从文件指针位置开始发送数据,在调用时需调用者自己调整文件指针位置 @param mime_type: 二进制数据流的MIME类型 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') if mime_type is None: mime_type = 'application/octet-stream' header['Content-Type'] = mime_type authorization = self.authorization('put', bucket, key, header) header['Authorization'] = authorization url = ufile_put_url(bucket, key) logger.info('start put stream to bucket {0} as {1}'.format( bucket, key)) logger.info('put UFile url: {0}'.format(url)) logger.info('request header:\n{0}'.format(json.dumps(header, indent=4))) return _put_stream(url, header, stream)
def putstream(self, bucket, key, stream, mime_type=None, header=None): """ 上传二进制流到空间 @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param stream: 二进制数据流,从文件指针位置开始发送数据,在调用时需调用者自己调整文件指针位置 @param mime_type: 二进制数据流的MIME类型 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') if mime_type is None: mime_type = 'application/octet-stream' header['Content-Type'] = mime_type authorization = self.authorization('put', bucket, key, header) header['Authorization'] = authorization url = ufile_put_url(bucket, key) logger.info('start put stream to bucket {0} as {1}'.format(bucket, key)) logger.info('put UFile url: {0}'.format(url)) logger.info('request header:\n{0}'.format(json.dumps(header, indent=4))) return _put_stream(url, header, stream)
def putfile(self, bucket, key, localfile, header=None): """ upload localfile to bucket as key @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param localfile: string类型,本地文件名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') mime_type = s(Mimetype.from_file(localfile)) file_size = os.path.getsize(localfile) header['Content-Type'] = mime_type authorization = self.authorization('put', bucket, key, header) header['Authorization'] = authorization header['Content-Length'] = file_size url = ufile_put_url(bucket, key) logger.info('start put file {0} to bucket {1} as {2}'.format(localfile, bucket, key)) logger.info('put UFile url: {0}'.format(url)) logger.info('request header:\n{0}'.format(json.dumps(header, indent=4))) return _put_file(url, header, localfile)
def deletebucket(self, bucket, header=None, projectid=None): """ 删除空间 @param bucket: string类型,空间名称 @param projectid: string类型,项目ID @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') param = dict() param['Action'] = 'DeleteBucket' param['BucketName'] = bucket if projectid is not None: param['ProjectId'] = projectid signature = self.__auth.bucket_signature(param) param['Signature'] = signature logger.info('start delete bucket {0}'.format(bucket)) return _bucket_request(UCLOUD_API_URL, param, header)
def ufile_uploadhit_url(bucket): """ 秒传UCloud UFile文件的url @param bucket: string类型, 待创建的空间名称 @return string类型, 秒传UFile的url """ return 'http://{0}{1}/uploadhit'.format(bucket, config.get_default('upload_suffix'))
def ufile_post_url(bucket): """ 采用表单上传方法上传UCloud UFile文件的url @param bucket: string类型, 待创建的空间名称 @return string类型, 表单上传UFile的url """ return 'http://{0}{1}/'.format(bucket, config.get_default('upload_suffix'))
def ufile_getfilelist_url(bucket): """ 获取文件列表的url @param bucket: string 类型,获取的空间名称 @return string类型,获取文件列表的url """ return 'http://{0}{1}/?list'.format(bucket, config.get_default('upload_suffix'))
def public_download_url(self, bucket, key): """ 从公共空间下载文件的url @param bucket: string类型, 空间名称 @param key: string类型,下载数据在空间中的名称 @return string类型,下载数据的url """ return 'http://{0}{1}/{2}'.format(bucket, config.get_default('download_suffix'), key)
def initialsharding_url(bucket, key): """ 初始化分片上传UCloud UFile的url @param bucket: string类型, 待创建的空间名称 @param key: string类型, 在空间中的文件名 @return string类型, 初始化分片上传UFile的url """ return 'http://{0}{1}/{2}?uploads'.format(bucket, config.get_default('upload_suffix'), key)
def ufile_put_url(bucket, key): """ 采用普通上传方法上传UCloud UFile文件的url @param bucket: string类型, 待创建的空间名称 @param key: string类型, 在空间中的文件名 @return string类型, 普通上传UFile的url """ return 'http://{0}{1}/{2}'.format(bucket, config.get_default('upload_suffix'), key)
def shardingupload_url(bucket, key, uploadid, part_number): """ 分片上传UCloud UFile的url @param bucket: string类型, 待创建的空间名称 @param key: string类型, 在空间中的文件名 @param uploadid: string类型, 初始化分片上传获得的uploadid字符串 @param part_number: integer类型, 分片上传的编号,从0开始 @return string类型, 结束分片上传UFile的url """ return 'http://{0}{1}/{2}?uploadId={3}&partNumber={4}'.format(bucket, config.get_default('upload_suffix'), key, uploadid, s(str(part_number)))
def test_downloadwithrange(self): self.downloadufile_handler.set_keys(public_key, private_key) logger.info('start download with range condition from public bucket') ret, resp = self.downloadufile_handler.download_file( public_bucket, put_range_key, public_range_download, isprivate=False, expires=get_default('expires'), content_range=(0, 5), header=None) assert resp.status_code == 206 logger.info('start download with range condition from private bucket') ret, resp = self.downloadufile_handler.download_file( public_bucket, put_range_key, private_range_download, isprivate=True, expires=get_default('expires'), content_range=(0, 5), header=None) assert resp.status_code == 206
def _delete_file(url, header): """ 删除文件 @param url: string类型, 要删除文件的url @param header: dict 类型,键值对类型分别为string类型,HTTP请求头 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.delete(url, headers=header, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def _put_stream(url, header, data): """ 采用普通方法上传二进制流到UFile空间 @param url: string类型,上传的url @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @param data: 二进制数据流 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.put(url, headers=header, data=data, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def _shardingupload(url, data, header): """ 分片上传数据 @param url: string类型,分片上传数据的url @param data: bytes 类型,分片上传的二进制数据 @param header: dict 类型,键值对类型分别为string类型,HTTP请求头 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.put(url, headers=header, data=data, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def _uploadhit_file(url, header, params): """ 秒传文件到空间 @param url:string类型, 秒传文件的url @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @param params: dict类型,http 请求的查询参数,键值对类型分别为string类型 @return ret: return message, None if response status code not in [200, 204, 206] else a dict-like object with response body @return ResponseInfo: UCloud UFile server response info """ try: response = requests.post(url, params=params, headers=header, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def _initialsharding(url, header): """ 初始化分片请求 @param url: string类型, 初始化分片请求的url @param header: dict 类型,键值对类型分别为string类型,HTTP请求头 @return ret: 如果http状态码不为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.post( url, headers=header, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def getfilelist(self, bucket, prefix=None, marker=None, limit=None, header=None): """ 获取bucket下的文件列表 @param bucket: string 类型,空间名称 @param prefix: string 类型,文件前缀, 默认为空字符串 @param marker: string 类型,文件列表起始位置, 默认为空字符串 @param limit: integer 类型,文件列表数目, 默认为20 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') header['Content-Length'] = 0 authorization = self.authorization('get', bucket, '', header) header['Authorization'] = authorization param = dict() if marker is not None and (isinstance(marker, str) or isinstance(marker, unicode)): param['marker'] = s(marker) if prefix is not None and (isinstance(prefix, str) or isinstance(prefix, unicode)): param['prefix'] = s(prefix) if limit is not None and isinstance(limit, int): param['limit'] = s(str(limit)) info_message = ''.join([ 'start get file list from bucket {0}'.format(bucket), '' if marker is None else ', marker: {0}'.format( marker if isinstance(marker, str) else marker.encode('utf-8')), '' if limit is None else ', limit: {0}'.format(limit), '' if prefix is None else ', prefix: {0}'.format(prefix) ]) logger.info(info_message) url = ufile_getfilelist_url(bucket) return _getfilelist(url, header, param)
def postfile(self, bucket, key, localfile, header=None): """ 表单上传文件到UFile空间 @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param localfile: string类型,本地文件名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') mime_type = s(Mimetype.from_file(localfile)) # update the request header content-type boundary = self.__make_boundary() header['Content-Type'] = 'multipart/form-data; boundary={0}'.format( boundary) # form fields authorization = self.authorization('post', bucket, key, header, mime_type) fields = dict() fields['FileName'] = key fields['Authorization'] = authorization with open(localfile, 'rb') as stream: postdata = self.__make_postbody(boundary, fields, stream, mime_type, localfile) # update the request header content-length header['Content-Length'] = str(len(postdata)) # post url url = ufile_post_url(bucket) # start post file logger.info('start post file {0} to bucket {1} as {2}'.format( localfile, bucket, key)) logger.info('post url is {0}'.format(url)) return _post_file(url, header, postdata)
def _put_stream(url, header, data): """ 采用普通方法上传二进制流到UFile空间 @param url: string类型,上传的url @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @param data: 二进制数据流 @return ret: 如果http状态码不为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.put( url, headers=header, data=data, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def _shardingupload(url, data, header): """ 分片上传数据 @param url: string类型,分片上传数据的url @param data: bytes 类型,分片上传的二进制数据 @param header: dict 类型,键值对类型分别为string类型,HTTP请求头 @return ret: 如果http状态码不为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.put( url, headers=header, data=data, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def _getfilelist(url, header, param): """ 获取文件列表 @param url: string 类型,获取文件列表的url @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码不为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.get( url, headers=header, params=param, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: logger.error('send request error:{0}'.format(e)) return None, ResponseInfo(None, e) return __return_wraper(response)
def _uploadhit_file(url, header, params): """ 秒传文件到空间 @param url:string类型, 秒传文件的url @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @param params: dict类型,http 请求的查询参数,键值对类型分别为string类型 @return ret: return message, None if response status code not in [200, 204, 206] else a dict-like object with response body @return ResponseInfo: UCloud UFile server response info """ try: response = requests.post( url, params=params, headers=header, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def postfile(self, bucket, key, localfile, header=None): """ 表单上传文件到UFile空间 @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param localfile: string类型,本地文件名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() _check_dict(header) if "User-Agent" not in header: header["User-Agent"] = config.get_default("user_agent") mime_type = s(Mimetype.from_file(localfile)) # update the request header content-type boundary = self.__make_boundary() header["Content-Type"] = "multipart/form-data; boundary={0}".format(boundary) # form fields authorization = self.authorization("post", bucket, key, header, mime_type) fields = dict() fields["FileName"] = key fields["Authorization"] = authorization with open(localfile, "rb") as stream: postdata = self.__make_postbody(boundary, fields, stream, mime_type, localfile) # update the request header content-length header["Content-Length"] = len(postdata) # post url url = ufile_post_url(bucket) # start post file logger.info("start post file {0} to bucket {1} as {2}".format(localfile, bucket, key)) logger.info("post url is {0}".format(url)) return _post_file(url, header, postdata)
def uploadhit(self, bucket, key, localfile, header=None): """ 尝试秒传文件到UFile空间 @param bucket: string类型,上传空间名称 @param key: string 类型,上传文件在空间中的名称 @param localfile: string类型,本地文件名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') filesize = os.path.getsize(localfile) fileetags = file_etag(localfile, BLOCKSIZE) mimetype = s(Mimetype.from_file(localfile)) # update request header header['Content-Type'] = mimetype header['Content-Length'] = 0 authorization = self.authorization('post', bucket, key, header) header['Authorization'] = authorization # parameter params = {'Hash': fileetags, 'FileName': key, 'FileSize': filesize} url = ufile_uploadhit_url(bucket) logger.info('start upload hit localfile {0} as {1} in bucket {2}'.format(localfile, key, bucket)) logger.info('request url: {0}'.format(url)) return _uploadhit_file(url, header, params)
def deletebucket(self, bucket, header=None): """ 删除空间 @param bucket: string类型,空间名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') param = dict() param['Action'] = 'DeleteBucket' param['BucketName'] = bucket signature = self.__auth.bucket_signature(param) param['Signature'] = signature logger.info('start delete bucket {0}'.format(bucket)) return _bucket_request(UCLOUD_API_URL, param, header)
def deletefile(self, bucket, key, header=None): """ 删除空间中文件方法 @param bucket: string类型, 空间名称 @param key: string类型, 被删除文件在空间中的名称 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if header is None: header = dict() else: _check_dict(header) if 'User-Agent' not in header: header['User-Agent'] = config.get_default('user_agent') authorization = self.authorization('delete', bucket, key, header) header['Authorization'] = authorization logger.info('start delete file {0} in bucket {1}'.format(key, bucket)) url = ufile_put_url(bucket, key) return _delete_file(url, header)
def _finishsharding(url, param, header, data): """ 结束分片上传请求 @param url: string类型, 初始化分片请求的url @param header: dict 类型,键值对类型分别为string类型,HTTP请求头 @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ try: response = requests.post(url, headers=header, params=param, data=data, timeout=config.get_default('connection_timeout')) except requests.RequestException as e: return None, ResponseInfo(None, e) return __return_wraper(response)
def test_downloadwithrange(self): self.downloadufile_handler.set_keys(public_key, private_key) logger.info('start download with range condition from public bucket') ret, resp = self.downloadufile_handler.download_file(public_bucket, put_range_key, public_range_download, isprivate=False, expires=get_default('expires'), content_range=(0, 5), header=None) assert resp.status_code == 206 logger.info('start download with range condition from private bucket') ret, resp = self.downloadufile_handler.download_file(public_bucket, put_range_key, private_range_download, isprivate=True, expires=get_default('expires'), content_range=(0, 5), header=None) assert resp.status_code == 206
def resumeuploadstream(self, retrycount=3, retryinterval=5, bucket=None, key=None, uploadid=None, blocksize=None, etaglist=None, stream=None, pausepartnumber=None, mime_type=None, header=None): """ 断点续传失败数据流的分片 可以在调用uploadstream失败后重新续传,也可以通过传递所有需要的参数续传 @param retrycount: integer 类型,分片重传次数 @param retryinterval: integer 类型,同个分片失败重传间隔,单位秒 @param bucket: string类型, 空间名称 @param key: string类型,文件或数据在空间中的名称 @param uploadid: string类型,初始化分片获得的uploadid @param blocksize: integer类型,分片大小 @param etaglist: list类型,元素为已经上传成功的分片的etag @param pausepartnumber: integer类型,第一个失败分片的编号(编号从0开始) @param stream: file-like对象或者二进制数据流,需要重新上传的数据 @param mime_type: string类型,上传数据的MIME类型 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ if bucket: self.__bucket = bucket if key: self.__key = key if uploadid: self.uploadid = uploadid if blocksize: self.blocksize = blocksize if stream: self.__stream = stream if etaglist: self.etaglist = etaglist if pausepartnumber: self.pausepartnumber = pausepartnumber if header: self.__header = header if mime_type is not None: self.__mimetype = mime_type elif self.__mimetype is None: self.__mimetype = 'application/octec-stream' if self.__header is None: self.__header = dict() else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') # initial sharding request if self.uploadid is None: ret, resp = self.__initialsharding() if resp.ok(): self.uploadid = ret.get('UploadId') self.blocksize = ret.get('BlkSize') logger.info('multipart upload id: {0}'.format(self.uploadid)) else: logger.error('multipar upload init failed. error message: {0}'.format(resp.error)) return ret, resp self.__header['Content-Type'] = self.__mimetype authorization = self.authorization('put', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization for data in _file_iter(self.__stream, self.blocksize): url = shardingupload_url(self.__bucket, self.__key, self.uploadid, self.pausepartnumber) ret = None resp = None for index in range(retrycount): logger.info('retry {0} time sharding upload sharing {0}'.format(index + 1, self.pausepartnumber)) logger.info('sharding url:{0}'.format(url)) ret, resp = _shardingupload(url, data, self.__header) if not resp.ok(): logger.error('failed {0} time when retry upload sharding {1},error message: {2}, uploadid: {3}'.format(index + 1, self.pausepartnumber, resp.error, self.uploadid)) if index < retrycount - 1: time.sleep(retryinterval) else: break if not resp.ok(): logger.error('retry upload sharding {0} failed, uploadid: {1}'.format(self.pausepartnumber, self.uploadid)) return ret, resp logger.info('retry upload sharding {0} succeed. etag: {1}, uploadid: {2}'.format(self.pausepartnumber, resp.etag, self.uploadid)) self.pausepartnumber += 1 self.etaglist.append(resp.etag) # finish sharding upload logger.info('start finish upload request') ret, resp = self.__finishupload() if not resp.ok(): logger.error('multipart upload failed. uploadid:{0}, pausepartnumber: {1}, key: {2} FAIL!!!'.format(self.uploadid, self.pausepartnumber, self.__key)) else: logger.info('mulitpart upload succeed. uploadid: {0}, key: {1} SUCCEED!!!'.format(self.uploadid, self.__key)) return ret, resp
def uploadstream(self, bucket, key, stream, retrycount=3, retryinterval=5, mime_type=None, header=None): """ 分片上传二进制数据流到UFile空间 @param bucket: 空间名称 @param key: 上传数据在空间中的名称 @param stream: file-like 对象或者二进制数据流 @param mime_type: 上传数据的MIME类型 @param retrycount: integer 类型,分片重传次数 @param retryinterval: integer 类型,同个分片失败重传间隔,单位秒 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ self.__bucket = bucket self.__key = key self.etaglist = [] self.uploadid = None self.blocksize = None self.__header = header self.__stream = stream self.pausepartnumber = 0 if self.__header is None: self.__header = dict() else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') # initial sharding request ret, resp = self.__initialsharding() if resp.ok(): self.uploadid = ret.get('UploadId') self.blocksize = ret.get('BlkSize') logger.info('multipart upload id: {0}'.format(self.uploadid)) else: logger.error('multipar upload init failed. error message: {0}'.format(resp.error)) return ret, resp # mulitple sharding upload if mime_type is None: if hasattr(self.__stream, 'seek') and hasattr(self.__stream, 'read'): self.__mimetype = s(Mimetype.from_buffer(self.__stream.read(1024))) self.__stream.seek(0, os.SEEK_SET) else: self.__mimetype = 'application/octec-stream' else: self.__mimetype = mime_type self.__header['Content-Type'] = self.__mimetype authorization = self.authorization('put', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization for data in _file_iter(self.__stream, self.blocksize): url = shardingupload_url(self.__bucket, self.__key, self.uploadid, self.pausepartnumber) ret = None resp = None for index in range(retrycount): logger.info('try {0} time sharding upload sharding {1}'.format(index + 1, self.pausepartnumber)) logger.info('sharding url:{0}'.format(url)) ret, resp = _shardingupload(url, data, self.__header) if not resp.ok(): logger.error('failed {0} time when upload sharding {1}.error message: {2}, uploadid: {3}'.format(index + 1, self.pausepartnumber, resp.error, self.uploadid)) if index < retrycount - 1: time.sleep(retryinterval) else: break if not resp.ok(): logger.error('upload sharding {0} failed. uploadid: {1}'.format(self.pausepartnumber, self.uploadid)) return ret, resp logger.info('upload sharding {0} succeed.etag:{1}, uploadid: {2}'.format(self.pausepartnumber, resp.etag, self.uploadid)) self.pausepartnumber += 1 self.etaglist.append(resp.etag) logger.info('start finish sharding request.') ret, resp = self.__finishupload() if not resp.ok(): logger.error('multipart upload failed. uploadid:{0}, pausepartnumber: {1}, key: {2} FAIL!!!'.format(self.uploadid, self.pausepartnumber, self.__key)) else: logger.info('mulitpart upload succeed. uploadid: {0}, key: {1} SUCCEED!!!'.format(self.uploadid, self.__key)) return ret, resp
def uploadstream(self, bucket, key, stream, retrycount=3, retryinterval=5, mime_type=None, header=None): """ 分片上传二进制数据流到UFile空间 @param bucket: 空间名称 @param key: 上传数据在空间中的名称 @param stream: file-like 对象或者二进制数据流 @param mime_type: 上传数据的MIME类型 @param retrycount: integer 类型,分片重传次数 @param retryinterval: integer 类型,同个分片失败重传间隔,单位秒 @param header: dict类型,http 请求header,键值对类型分别为string,比如{'User-Agent': 'Google Chrome'} @return ret: 如果http状态码为[200, 204, 206]之一则返回None,否则如果服务器返回json信息则返回dict类型,键值对类型分别为string, unicode string类型,否则返回空的dict @return ResponseInfo: 响应的具体信息,UCloud UFile 服务器返回信息或者网络链接异常 """ self.__bucket = bucket self.__key = key self.etaglist = [] self.uploadid = None self.blocksize = None self.__header = header self.__stream = stream self.pausepartnumber = 0 if self.__header is None: self.__header = dict() else: _check_dict(self.__header) if 'User-Agent' not in self.__header: self.__header['User-Agent'] = config.get_default('user_agent') # initial sharding request ret, resp = self.__initialsharding() if resp.ok(): self.uploadid = ret.get('UploadId') self.blocksize = ret.get('BlkSize') logger.info('multipart upload id: {0}'.format(self.uploadid)) else: logger.error( 'multipar upload init failed. error message: {0}'.format( resp.error)) return ret, resp # mulitple sharding upload if mime_type is None: if hasattr(self.__stream, 'seek') and hasattr( self.__stream, 'read'): self.__mimetype = s( Mimetype.from_buffer(self.__stream.read(1024))) self.__stream.seek(0, os.SEEK_SET) else: self.__mimetype = 'application/octec-stream' else: self.__mimetype = mime_type self.__header['Content-Type'] = self.__mimetype authorization = self.authorization('put', self.__bucket, self.__key, self.__header) self.__header['Authorization'] = authorization for data in _file_iter(self.__stream, self.blocksize): url = shardingupload_url(self.__bucket, self.__key, self.uploadid, self.pausepartnumber) ret = None resp = None for index in range(retrycount): logger.info('try {0} time sharding upload sharding {1}'.format( index + 1, self.pausepartnumber)) logger.info('sharding url:{0}'.format(url)) ret, resp = _shardingupload(url, data, self.__header) if not resp.ok(): logger.error( 'failed {0} time when upload sharding {1}.error message: {2}, uploadid: {3}' .format(index + 1, self.pausepartnumber, resp.error, self.uploadid)) if index < retrycount - 1: time.sleep(retryinterval) else: break if not resp.ok(): logger.error( 'upload sharding {0} failed. uploadid: {1}'.format( self.pausepartnumber, self.uploadid)) return ret, resp logger.info( 'upload sharding {0} succeed.etag:{1}, uploadid: {2}'.format( self.pausepartnumber, resp.etag, self.uploadid)) self.pausepartnumber += 1 self.etaglist.append(resp.etag) logger.info('start finish sharding request.') ret, resp = self.__finishupload() if not resp.ok(): logger.error( 'multipart upload failed. uploadid:{0}, pausepartnumber: {1}, key: {2} FAIL!!!' .format(self.uploadid, self.pausepartnumber, self.__key)) else: logger.info( 'mulitpart upload succeed. uploadid: {0}, key: {1} SUCCEED!!!'. format(self.uploadid, self.__key)) return ret, resp