def list_objects(self, Bucket, Delimiter="", Marker="", MaxKeys=1000, Prefix="", **kwargs): """获取文件列表""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("list objects, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) params = { 'delimiter': Delimiter, 'marker': Marker, 'max-keys': MaxKeys, 'prefix': Prefix} rt = self.send_request( method='GET', url=url, params=params, headers=headers, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) data = xml_to_dict(rt.text) if 'Contents' in data.keys() and isinstance(data['Contents'], dict): # 只有一个Contents,将dict转为list,保持一致 lst = [] lst.append(data['Contents']) data['Contents'] = lst return data
def init_multiupload(): url = self._conf.uri(path=cos_path) self._md5 = {} self._have_finished = 0 self._have_uploaded = [] self._upload_id = None self._path_md5 = get_md5_filename(local_path, cos_path) logger.debug("init with : " + url) if os.path.isfile(self._path_md5): with open(self._path_md5, 'rb') as f: self._upload_id = f.read() if self.list_part(cos_path) is True: logger.info("continue uploading from last breakpoint") return True rt = self._session.post(url=url + "?uploads", auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) logger.debug( "init resp, status code: {code}, headers: {headers}, text: {text}" .format(code=rt.status_code, headers=rt.headers, text=rt.text)) if rt.status_code == 200: root = minidom.parseString(rt.content).documentElement self._upload_id = root.getElementsByTagName( "UploadId")[0].childNodes[0].data if os.path.isdir(os.path.expanduser("~/.tmp")) is False: os.makedirs(os.path.expanduser("~/.tmp")) with open(self._path_md5, 'wb') as f: f.write(self._upload_id) return True else: logger.warn(response_info(rt)) return False return True
def single_upload(): if len(local_path) == 0: data = "" else: with open(local_path, 'rb') as File: data = File.read() url = self._conf.uri(path=cos_path) for j in range(self._retry): try: rt = self._session.put(url=url, auth=CosS3Auth( self._conf._access_id, self._conf._access_key), data=data) if rt.status_code == 200: if local_path != '': logger.info("upload {file} with {per}%".format( file=to_printable_str(local_path), per="{0:5.2f}".format(100))) return True else: time.sleep(2**j) logger.warn(response_info(rt)) continue if j + 1 == self._retry: return False except Exception as e: logger.warn("upload file failed") return False
def list_parts(self, Bucket, Key, UploadId, EncodingType='url', MaxParts=1000, PartNumberMarker=0, **kwargs): """列出已上传的分片""" headers = mapped(kwargs) params = { 'uploadId': UploadId, 'part-number-marker': PartNumberMarker, 'max-parts': MaxParts, 'encoding-type': EncodingType } url = self._conf.uri(bucket=Bucket, path=Key) logger.info( "list multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='GET', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers, params=params) data = xml_to_dict(rt.text) if 'Part' in data.keys() and isinstance( data['Part'], dict): # 只有一个part,将dict转为list,保持一致 lst = [] lst.append(data['Part']) data['Part'] = lst return data
def sign_url(self, cos_path, timeout=10000): url = self._conf.uri(path=cos_path) s = requests.Session() req = requests.Request('GET', url) prepped = s.prepare_request(req) signature = CosS3Auth( self._conf._access_id, self._conf._access_key, timeout).__call__(prepped).headers['Authorization'] return url + '?sign=' + urllib.quote(signature)
def download_folder(self, cos_path, local_path, _force=False): def download_file(_cos_path, _local_path, _force): if self.download_file(_cos_path, _local_path, _force) is True: logger.info( "download {file}".format(file=to_printable_str(_cos_path))) self._have_finished += 1 else: logger.info("download {file} fail".format( file=to_printable_str(_cos_path))) self._fail_num += 1 NextMarker = "" IsTruncated = "true" self._file_num = 0 self._have_finished = 0 self._fail_num = 0 cos_path = to_unicode(cos_path) while IsTruncated == "true": url = self._conf.uri( path='?prefix={prefix}&marker={nextmarker}'.format( prefix=to_printable_str(cos_path), nextmarker=to_printable_str(NextMarker))) rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) if rt.status_code == 200: root = minidom.parseString(rt.content).documentElement IsTruncated = root.getElementsByTagName( "IsTruncated")[0].childNodes[0].data if IsTruncated == 'true': NextMarker = root.getElementsByTagName( "NextMarker")[0].childNodes[0].data fileset = root.getElementsByTagName("Contents") for _file in fileset: self._file_num += 1 _cos_path = _file.getElementsByTagName( "Key")[0].childNodes[0].data _local_path = local_path + _cos_path[len(cos_path):] _cos_path = to_unicode(_cos_path) _local_path = to_unicode(_local_path) download_file(_cos_path, _local_path, _force) else: logger.warn(response_info(rt)) return False if self._file_num == 0: logger.info("The directory does not exist") return False logger.info( "{files} files successful, {fail_files} files failed".format( files=self._have_finished, fail_files=self._fail_num)) if self._file_num == self._have_finished: return True else: return False
def delete_object(self, Bucket, Key, **kwargs): """单文件删除接口""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) logger.info("delete object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='DELETE', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None
def delete_bucket(self, Bucket, **kwargs): """删除一个bucket,bucket必须为空""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("delete bucket, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='DELETE', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None
def head_bucket(self, Bucket, **kwargs): """获取bucket信息""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket) logger.info("head bucket, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='HEAD', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None
def put_bucket_acl(self, Bucket, **kwargs): """设置bucket ACL""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?acl") logger.info("put bucket acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='PUT', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None
def get_auth(self, Method, Bucket, Key=None, Expired=300, headers={}, params={}): """获取签名""" url = self._conf.uri(bucket=Bucket, path=Key) r = Request(Method, url, headers=headers, params=params) auth = CosS3Auth(self._conf._access_id, self._conf._access_key, Expired) return auth(r).headers['Authorization']
def abort_multipart_upload(self, Bucket, Key, UploadId, **kwargs): """放弃一个已经存在的分片上传任务,删除所有已经存在的分片""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key + "?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("abort multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='DELETE', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) return None
def create_multipart_upload(self, Bucket, Key, **kwargs): """创建分片上传,适用于大文件上传""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key + "?uploads") logger.info("create multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) data = xml_to_dict(rt.text) return data
def list_buckets(self, **kwargs): """列出所有bucket""" headers = mapped(kwargs) url = 'http://service.cos.myqcloud.com/' rt = self.send_request( method='GET', url=url, headers=headers, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), ) data = xml_to_dict(rt.text) if data['Buckets'] is not None and isinstance(data['Buckets']['Bucket'], dict): lst = [] lst.append(data['Buckets']['Bucket']) data['Buckets']['Bucket'] = lst return data
def copy_object(self, Bucket, Key, CopySource, CopyStatus='Copy', **kwargs): """文件拷贝,文件信息修改""" headers = mapped(kwargs) headers['x-cos-copy-source'] = self.gen_copy_source_url(CopySource) headers['x-cos-metadata-directive'] = CopyStatus url = self._conf.uri(bucket=Bucket, path=Key) logger.info("copy object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='PUT', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) data = xml_to_dict(rt.text) return data
def complete_multipart_upload(self, Bucket, Key, UploadId, MultipartUpload={}, **kwargs): """完成分片上传,组装后的文件不得小于1MB,否则会返回错误""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key + "?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("complete multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='POST', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=dict_to_xml(MultipartUpload), timeout=1200, # 分片上传大文件的时间比较长,设置为20min headers=headers) data = xml_to_dict(rt.text) return data
def download_file(self, cos_path, local_path, _force=False): if _force is False and os.path.isfile(local_path) is True: logger.warn( "The file {file} already exists, please use -f to overwrite the file" .format(file=to_printable_str(cos_path))) return False url = self._conf.uri(path=cos_path) logger.debug("download with : " + url) try: rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), stream=True) logger.debug( "get resp, status code: {code}, headers: {headers}".format( code=rt.status_code, headers=rt.headers)) if 'Content-Length' in rt.headers: content_len = int(rt.headers['Content-Length']) else: raise IOError("download failed without Content-Length header") if rt.status_code == 200: self._pbar = tqdm(total=content_len, unit='B', unit_scale=True) file_len = 0 dir_path = os.path.dirname(local_path) if os.path.isdir(dir_path) is False and dir_path != '': try: os.makedirs(dir_path) except Exception as e: logger.warn(str(e)) return False with open(local_path, 'wb') as f: for chunk in rt.iter_content(chunk_size=1024): if chunk: self._pbar.update(len(chunk)) file_len += len(chunk) f.write(chunk) f.flush() if file_len != content_len: raise IOError("download failed with incomplete file") self._pbar.close() return True else: logger.warn(response_info(rt)) return False except Exception as e: logger.warn(str(e)) return False
def get_bucket(self, max_keys=10): NextMarker = "" IsTruncated = "true" pagecount = 0 filecount = 0 sizecount = 0 while IsTruncated == "true": pagecount += 1 logger.info("get bucket with page {page}".format(page=pagecount)) url = self._conf.uri( path='?max-keys=1000&marker={nextmarker}'.format( nextmarker=NextMarker)) rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) if rt.status_code == 200: root = minidom.parseString(rt.content).documentElement IsTruncated = root.getElementsByTagName( "IsTruncated")[0].childNodes[0].data if IsTruncated == 'true': NextMarker = root.getElementsByTagName( "NextMarker")[0].childNodes[0].data logger.debug( "init resp, status code: {code}, headers: {headers}, text: {text}" .format(code=rt.status_code, headers=rt.headers, text=to_printable_str(rt.text))) contentset = root.getElementsByTagName("Contents") for content in contentset: filecount += 1 sizecount += int( content.getElementsByTagName("Size") [0].childNodes[0].data) print to_printable_str(content.toxml()) if filecount == max_keys: break else: logger.warn(response_info(rt)) return False logger.info("filecount: %d" % filecount) logger.info("sizecount: %d" % sizecount) logger.debug("get bucket success") return True
def get_bucket_acl(self, Bucket, **kwargs): """获取bucket ACL""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path="?acl") logger.info("get bucket acl, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) data = xml_to_dict(rt.text) if data['AccessControlList'] is not None and isinstance(data['AccessControlList']['Grant'], dict): lst = [] lst.append(data['AccessControlList']['Grant']) data['AccessControlList']['Grant'] = lst return data
def list_parts(self, Bucket, Key, UploadId, **kwargs): """列出已上传的分片""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key + "?uploadId={UploadId}".format(UploadId=UploadId)) logger.info("list multipart upload, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='GET', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) data = xml_to_dict(rt.text) if 'Part' in data.keys() and isinstance(data['Part'], dict): # 只有一个part,将dict转为list,保持一致 lst = [] lst.append(data['Part']) data['Part'] = lst return data
def upload_part(self, Bucket, Key, Body, PartNumber, UploadId, **kwargs): """上传分片,单个大小不得超过5GB""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key + "?partNumber={PartNumber}&uploadId={UploadId}".format( PartNumber=PartNumber, UploadId=UploadId)) logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request( method='PUT', url=url, headers=headers, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=Body) response = dict() response['ETag'] = rt.headers['ETag'] return response
def delete_file(self, cos_path): url = self._conf.uri(path=cos_path) logger.info("delete with : " + url) try: rt = self._session.delete(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) logger.debug( "init resp, status code: {code}, headers: {headers}".format( code=rt.status_code, headers=rt.headers)) if rt.status_code == 204 or rt.status_code == 200: return True else: logger.warn(response_info(rt)) return False except Exception as e: logger.warn(str(e)) return False return False
def get_object(self, Bucket, Key, **kwargs): """单文件下载接口""" headers = mapped(kwargs) url = self._conf.uri(bucket=Bucket, path=Key) logger.info("get object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='GET', url=url, stream=True, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), headers=headers) response = dict() response['Body'] = StreamBody(rt) for k in rt.headers.keys(): response[k] = rt.headers[k] return response
def delete_bucket(self): url = self._conf.uri(path='') self._have_finished = 0 logger.debug("delete bucket with : " + url) try: rt = self._session.delete(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) logger.debug( "delete resp, status code: {code}, headers: {headers}, text: {text}" .format(code=rt.status_code, headers=rt.headers, text=rt.text)) if rt.status_code == 204: return True else: logger.warn(response_info(rt)) return False except Exception as e: logger.warn(str(e)) return False return True
def complete_multiupload(): logger.info('completing multiupload') doc = minidom.Document() root = doc.createElement("CompleteMultipartUpload") list_md5 = sorted(self._md5.items(), key=lambda d: d[0]) for i, v in list_md5: t = doc.createElement("Part") t1 = doc.createElement("PartNumber") t1.appendChild(doc.createTextNode(str(i))) t2 = doc.createElement(self._etag) t2.appendChild(doc.createTextNode('"{v}"'.format(v=v))) t.appendChild(t1) t.appendChild(t2) root.appendChild(t) data = root.toxml() url = self._conf.uri( path=cos_path) + "?uploadId={uploadid}".format( uploadid=self._upload_id) logger.debug('complete url: ' + url) logger.debug("complete data: " + data) try: with closing( self._session.post(url, auth=CosS3Auth( self._conf._access_id, self._conf._access_key), data=data, stream=True)) as rt: logger.debug("complete status code: {code}".format( code=rt.status_code)) logger.debug("complete headers: {headers}".format( headers=rt.headers)) if rt.status_code == 200: os.remove(self._path_md5) return True else: logger.warn(response_info(rt)) return False except Exception as e: return False return True
def put_object(self, Bucket, Body, Key, **kwargs): """单文件上传接口,适用于小文件,最大不得超过5GB""" headers = mapped(kwargs) if 'Metadata' in headers.keys(): for i in headers['Metadata'].keys(): headers[i] = headers['Metadata'][i] headers.pop('Metadata') url = self._conf.uri(bucket=Bucket, path=Key) logger.info("put object, url=:{url} ,headers=:{headers}".format( url=url, headers=headers)) rt = self.send_request(method='PUT', url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=Body, headers=headers) response = dict() response['ETag'] = rt.headers['ETag'] return response
def multiupload_parts_data(local_path, offset, length, parts_size, idx): with open(local_path, 'rb') as File: File.seek(offset, 0) data = File.read(length) url = self._conf.uri( path=cos_path ) + "?partNumber={partnum}&uploadId={uploadid}".format( partnum=idx, uploadid=self._upload_id) logger.debug("upload url: " + str(url)) for j in range(self._retry): rt = self._session.put(url=url, auth=CosS3Auth( self._conf._access_id, self._conf._access_key), data=data) logger.debug( "multi part result: part{part}, round{round}, code: {code}, headers: {headers}, text: {text}" .format(part=idx, round=j + 1, code=rt.status_code, headers=rt.headers, text=rt.text)) self._md5[idx] = rt.headers[self._etag][1:-1] if rt.status_code == 200: self._have_finished += 1 self._pbar.update(length) break else: logger.warn(response_info(rt)) time.sleep(2**j) continue if j + 1 == self._retry: logger.warn( "upload part failed: part{part}, round{round}, code: {code}" .format(part=idx, round=j + 1, code=rt.status_code)) return False return True
def list_part(self, cos_path): logger.debug("getting uploaded parts") NextMarker = "" IsTruncated = "true" cos_path = to_printable_str(cos_path) while IsTruncated == "true": url = self._conf.uri( path=cos_path + '?uploadId={UploadId}&upload&max-parts=1000&part-number-marker={nextmarker}' .format(UploadId=self._upload_id, nextmarker=NextMarker)) rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) if rt.status_code == 200: root = minidom.parseString(rt.content).documentElement IsTruncated = root.getElementsByTagName( "IsTruncated")[0].childNodes[0].data if IsTruncated == 'true': NextMarker = root.getElementsByTagName( "NextPartNumberMarker")[0].childNodes[0].data logger.debug( "list resp, status code: {code}, headers: {headers}, text: {text}" .format(code=rt.status_code, headers=rt.headers, text=to_printable_str(rt.text))) contentset = root.getElementsByTagName("Part") for content in contentset: ID = content.getElementsByTagName( "PartNumber")[0].childNodes[0].data self._have_uploaded.append(ID) self._md5[int(ID)] = content.getElementsByTagName( self._etag)[0].childNodes[0].data[1:-1] else: logger.debug(response_info(rt)) return False logger.debug("list parts error") return True
def get_object_acl(self, cos_path): url = self._conf.uri(cos_path + "?acl") logger.info("get with : " + url) table = PrettyTable([cos_path, ""]) table.align = "l" table.padding_width = 3 try: rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) logger.debug( "get resp, status code: {code}, headers: {headers}".format( code=rt.status_code, headers=rt.headers)) root = minidom.parseString(rt.content).documentElement grants = root.getElementsByTagName("Grant") for grant in grants: table.add_row([ 'ACL', ("%s: %s" % (grant.getElementsByTagName("ID")[0].childNodes[0].data, grant.getElementsByTagName("Permission") [0].childNodes[0].data)) ]) if rt.status_code == 200: try: print unicode(table) except Exception as e: print table return True else: logger.warn(response_info(rt)) return False except Exception as e: logger.warn(str(e)) return False return False
def put_object_acl(self, grant_read, grant_write, grant_full_control, cos_path): acl = [] if grant_read is not None: for i in grant_read.split(","): if len(i) > 0: acl.append([i, "READ"]) if grant_write is not None: for i in grant_write.split(","): if len(i) > 0: acl.append([i, "WRITE"]) if grant_full_control is not None: for i in grant_full_control.split(","): if len(i) > 0: acl.append([i, "FULL_CONTROL"]) url = self._conf.uri(cos_path + "?acl") logger.info("put with : " + url) try: rt = self._session.get(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key)) if rt.status_code != 200: logger.warn(response_info(rt)) return False root = minidom.parseString(rt.content).documentElement owner_id = root.getElementsByTagName("ID")[0].childNodes[0].data grants = '' subid = '' rootid = '' for ID, Type in acl: if len(ID.split("/")) == 1: accounttype = "RootAccount" rootid = ID.split("/")[0] subid = ID.split("/")[0] elif len(ID.split("/")) == 2: accounttype = "SubAccount" rootid = ID.split("/")[0] subid = ID.split("/")[1] else: logger.warn("ID format error!") return False if subid != "anyone": subid = "uin/" + subid rootid = "uin/" + rootid grants += ''' <Grant> <Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="{accounttype}"> <ID>qcs::cam::{rootid}:{subid}</ID> </Grantee> <Permission>{permissiontype}</Permission> </Grant>'''.format(rootid=rootid, subid=subid, accounttype=accounttype, permissiontype=Type) data = '''<AccessControlPolicy> <Owner> <ID>{id}</ID> </Owner> <AccessControlList>'''.format(id=owner_id) + grants + ''' </AccessControlList> </AccessControlPolicy> ''' logger.debug(data) rt = self._session.put(url=url, auth=CosS3Auth(self._conf._access_id, self._conf._access_key), data=data) logger.debug( "put resp, status code: {code}, headers: {headers}".format( code=rt.status_code, headers=rt.headers)) if rt.status_code == 200: return True else: logger.warn(response_info(rt)) return False except Exception as e: logger.warn(str(e)) return False return False