def make_request(self, method, bucket, key, path_args, headers, s3_object): calling_format = Utils.get_callingformat_for_bucket(self.calling_format, bucket) if self.is_secure and not isinstance(calling_format, PathFormat) and bucket.find( "." ) != -1: raise Exception("You are making an SSL connection, however, the bucket contains periods and \ the wildcard certificate will not match by default. Please consider using HTTP.") path = calling_format.get_url(self.is_secure, self.server, self.port, bucket, key, path_args) connect_server = calling_format.get_server(self.server, bucket) if s3_object: head = self.add_metadata_headers(self.add_headers(headers, ""), s3_object.metadata) else: head = self.add_headers(headers, "") headerconfig = self.add_auth_headers(head, method, bucket, key, path_args) if s3_object: if s3_object.file_path: headerconfig["Content-Length"] = str(os.path.getsize(Utils.decode_utf(s3_object.file_path))) #需要手动将content-length添加到头信息中 else: headerconfig["Content-Length"] = str(0) return self.send_request(connect_server, method, path, headerconfig)
def add_metadata_headers(self, headers, metadata): if metadata: for i in metadata.keys(): key = str(i) for v in metadata.get(key): value = str(v) headers[(Utils.METADATA_PREFIX + Utils.urlencode(key))] = Utils.urlencode(value) return headers
def add_headers(self, headers, prefix): new_headers = {} if headers: for i in headers.keys(): key = str(i) for v in headers.get(key): value = str(v) new_headers[(prefix + Utils.urlencode(key))] = Utils.urlencode(value) return new_headers
def create_object(self, bucket, key, s3_object, headers = None): conn = self.make_request("PUT", bucket, Utils.urlencode(key), None, headers, s3_object) CHUNKSIZE = 65563 if s3_object.file_path: with open(Utils.decode_utf(s3_object.file_path), 'rb') as f: while True: chunk = f.read(CHUNKSIZE) if not chunk: break conn.send(chunk) return conn.getresponse()
def make_bare_url(self, bucket, key): buf = ["https://" if self.is_secure else "http://"] buf.append(self.server + ":" + str(self.port) + "/" + bucket) buf.append("/" + Utils.urlencode(key)) return ''.join(item for item in buf)
def get_object(self, bucket, key, headers = None): conn = self.make_request("GET", bucket, Utils.urlencode(key), None, headers, None) result = conn.getresponse() if int(result.status) < 400: response = GetResponse.get_object_factory(result) return response
def add_auth_headers(self, headers, method, bucket, key, path_args): #如果headers头中不存在Date和Content-Type或其值为空时,则添加 if not headers.get("Date"): headers["Date"] = self.httpdate() if not headers.get("Content-Type"): headers["Content-Type"] = "" canonical_string = Utils.make_canonicalstring(method, bucket, key, path_args, headers, None) web.debug("canonical_string is %s" % canonical_string) encoded_canonical = Utils.encode(self.secret_access_key, canonical_string, False) auth = "AWS " + self.access_key_id + ":" + encoded_canonical #字符串连接 headers["Authorization"] = auth return headers
def create_bucket(self, bucket, headers = None): #验证存储空间名字是否合法 if not Utils.validate_bucketname(bucket, self.calling_format): raise Exception("Invalid Bucket Name: " + bucket) conn = self.make_request("PUT", bucket, "", None, headers, None) response = conn.getresponse() return response
def list_objects(self, bucket, prefix = None, marker = None, max_keys = None, delimiter = None, headers = None): path_args = Utils.params_for_dict_options(prefix, marker, max_keys, delimiter) conn = self.make_request("GET", bucket, "", path_args, headers, None) result = conn.getresponse() if int(result.status) < 400: xml = result.read() response = ListObjectsResponse.list_objects_factory(xml) return response
def get_object_acl(self, bucket, key, headers): if key == None: key = "" path_args = {} path_args["acl"] = None conn = self.make_request("GET", bucket, Utils.urlencode(key), path_args, headers, None) result = conn.getresponse() if int(result.status) < 400: return result.read()
def generate_url(self, method, bucket, key, path_args, headers): expires = long(0) if self.expiresin: expires = long(round(time.time() * 1000)) + long(self.expiresin) #time.time()*1000获取当前时间的毫秒数 elif self.expires: expires = long(self.expires) else: raise Exception("Illegal expires state") #单位转换为秒 expires /= 1000 canonical_string = Utils.make_canonicalstring(method, bucket, key, path_args, headers, str(expires)) encoded_canonical = Utils.encode(self.secret_access_key, canonical_string, True) path_args = path_args if path_args else {} path_args["Signature"] = encoded_canonical path_args["Expires"] = str(expires) #将long型转换为string型 path_args["AWSAccessKeyId"] = self.access_key_id calling_format = Utils.get_callingformat_for_bucket(self.calling_format, bucket) if self.is_secure and not isinstance(calling_format, PathFormat) and bucket.find( "." ) != -1: raise Exception("You are making an SSL connection, however, the bucket contains periods and \ the wildcard certificate will not match by default. Please consider using HTTP.") #需要进行异常检查 try: return_string = calling_format.get_full_url(self.is_secure, self.server, self.port, bucket, key, path_args) except Exception as e: return_string = "Exception generating url " + e.strerror return return_string
def parse_buckets(xml): root = ET.fromstring(xml) buckets = root.find('{0}Buckets'.format(ListAllMyBuckets.NS)).findall('{0}Bucket'.format(ListAllMyBuckets.NS)) entries = [] for bucket in buckets: name = bucket.find("{0}Name".format(ListAllMyBuckets.NS)).text #获取bucket的名称 d = bucket.find("{0}CreationDate".format(ListAllMyBuckets.NS)).text #获取bucket的创建日期 creation_date = Utils.transfer_date(d) #将创建日期转换为当地时间 curr_bucket = Bucket(name, creation_date) #创建bucket对象 entries.append(curr_bucket) #向entries列表中添加bucket对象 return entries
def get_acl_response(self, bucket, key, headers = None): if key == None: key = "" path_args = {} path_args["acl"] = None conn = self.make_request("GET", bucket, Utils.urlencode(key), path_args, headers, None) result = conn.getresponse() if int(result.status) < 400: xml = result.read() response = ACLResponse.acl_factory(xml) return response
def load_xml_file(self, xml): root = ET.fromstring(xml) #获取xml文件的根节点root self.name = self.find_item(root, '{0}Name'.format(ListObjects.NS)) self.prefix = self.find_item(root, '{0}Prefix'.format(ListObjects.NS)) self.marker = self.find_item(root, '{0}Marker'.format(ListObjects.NS)) self.delimiter = self.find_item(root, '{0}Delimiter'.format(ListObjects.NS)) self.max_keys = self.find_item(root, '{0}MaxKeys'.format(ListObjects.NS)) is_truncated = self.find_item(root, '{0}IsTruncated'.format(ListObjects.NS)) self.is_truncated = self.convert_bool_value(is_truncated) self.next_marker = self.find_item( root, '{0}NextMarker'.format(ListObjects.NS)) #获取对象key的相关信息,在Contents节点中 contents = root.findall('{0}Contents'.format(ListObjects.NS)) if contents is not None: for node in contents: key = self.find_item(node, '{0}Key'.format(ListObjects.NS)) t = self.find_item(node, '{0}LastModified'.format(ListObjects.NS)) lastmodified = Utils.transfer_date(t) etag = self.find_item(node, '{0}ETag'.format(ListObjects.NS)) size = long( self.find_item(node, '{0}Size'.format(ListObjects.NS))) #获取Owner相关信息 owner_id = self.find_item(node, './/{0}ID'.format(ListObjects.NS)) owner_name = self.find_item( node, './/{0}DisplayName'.format(ListObjects.NS)) owner = Owner(owner_id, owner_name) #创建Owner对象 key_entry = ObjectEntry(key, lastmodified, etag, size, owner) self.key_entries.append(key_entry) #将对象添加到对象列表中 self.keyslist.append(key_entry.key) #将对象名添加到列表中 #获取CommonPrefixes的相关信息 prefixes = root.findall('{0}CommonPrefixes'.format(ListObjects.NS)) if prefixes is not None: for p in prefixes: pre = self.find_item(p, '{0}Prefix'.format(ListObjects.NS)) commonprefix = CommonPrefix(pre) self.commonprefix_entries.append(commonprefix)
def make_request2(self, method, bucket, key, data, headers): calling_format = Utils.get_callingformat_for_bucket(self.calling_format, bucket) if self.is_secure and not isinstance(calling_format, PathFormat) and bucket.find(".") != -1: raise Exception("You are making an SSL connection, however, the bucket contains periods and \ the wildcard certificate will not match by default. Please consider using HTTP.") web.debug("type of calling_format is %s" % calling_format) path = calling_format.get_url(self.is_secure, self.server, self.port, bucket, key, None) #path = '/galneryus' + path web.debug("path is %s" % path) connect_server = calling_format.get_server(self.server, bucket) head = self.add_headers(headers, "") header_config = self.add_auth_headers(head, method, bucket, key, None) header_config["Content-Length"] = str(len(data)) web.debug("header_config is %s" %str(header_config)) return self.send_request(connect_server, method, path, header_config)
def parse_buckets(xml): root = ET.fromstring(xml) buckets = root.find('{0}Buckets'.format(ListAllMyBuckets.NS)).findall( '{0}Bucket'.format(ListAllMyBuckets.NS)) entries = [] for bucket in buckets: name = bucket.find("{0}Name".format( ListAllMyBuckets.NS)).text #获取bucket的名称 d = bucket.find("{0}CreationDate".format( ListAllMyBuckets.NS)).text #获取bucket的创建日期 creation_date = Utils.transfer_date(d) #将创建日期转换为当地时间 curr_bucket = Bucket(name, creation_date) #创建bucket对象 entries.append(curr_bucket) #向entries列表中添加bucket对象 return entries
def set_object_acl(self, bucket, key, acl_path, headers): obj = S3Object(acl_path, None) path_args = {} path_args["acl"] = None conn = self.make_request("PUT", bucket, Utils.urlencode(key), path_args, headers, obj) CHUNKSIZE = 65563 if acl_path: with open(acl_path, 'rb') as f: while True: chunk = f.read(CHUNKSIZE) if not chunk: break conn.send(chunk) return conn.getresponse()
def load_xml_file(self, xml): root = ET.fromstring(xml) #获取xml文件的根节点root self.name = self.find_item(root, '{0}Name'.format(ListObjects.NS)) self.prefix = self.find_item(root, '{0}Prefix'.format(ListObjects.NS)) self.marker = self.find_item(root, '{0}Marker'.format(ListObjects.NS)) self.delimiter = self.find_item(root, '{0}Delimiter'.format(ListObjects.NS)) self.max_keys = self.find_item(root, '{0}MaxKeys'.format(ListObjects.NS)) is_truncated = self.find_item(root, '{0}IsTruncated'.format(ListObjects.NS)) self.is_truncated = self.convert_bool_value(is_truncated) self.next_marker = self.find_item(root, '{0}NextMarker'.format(ListObjects.NS)) #获取对象key的相关信息,在Contents节点中 contents = root.findall('{0}Contents'.format(ListObjects.NS)) if contents is not None: for node in contents: key = self.find_item(node, '{0}Key'.format(ListObjects.NS)) t = self.find_item(node, '{0}LastModified'.format(ListObjects.NS)) lastmodified = Utils.transfer_date(t) etag = self.find_item(node, '{0}ETag'.format(ListObjects.NS)) size = long(self.find_item(node, '{0}Size'.format(ListObjects.NS))) #获取Owner相关信息 owner_id = self.find_item(node, './/{0}ID'.format(ListObjects.NS)) owner_name = self.find_item(node, './/{0}DisplayName'.format(ListObjects.NS)) owner = Owner(owner_id, owner_name) #创建Owner对象 key_entry = ObjectEntry(key, lastmodified, etag, size, owner) self.key_entries.append(key_entry) #将对象添加到对象列表中 self.keyslist.append(key_entry.key) #将对象名添加到列表中 #获取CommonPrefixes的相关信息 prefixes = root.findall('{0}CommonPrefixes'.format(ListObjects.NS)) if prefixes is not None: for p in prefixes: pre = self.find_item(p, '{0}Prefix'.format(ListObjects.NS)) commonprefix = CommonPrefix(pre) self.commonprefix_entries.append(commonprefix)
def get_acl(self, bucket, key, headers): path_args = {} path_args["acl"] = None return self.generate_url("GET", bucket, Utils.urlencode(key), path_args, headers)
def get(self, bucket, key, headers): urlstr = self.generate_url("GET", bucket, Utils.urlencode(key), None, headers) return urlstr
def delete_object(self, bucket, key, headers): conn = self.make_request("DELETE", bucket, Utils.urlencode(key), None, headers, None) return conn.getresponse()
def create_object2(self, bucket, key, data, headers=None): conn = self.make_request2("PUT", bucket, Utils.urlencode(key), data, headers) conn.send(data) return conn.getresponse()