def upload_file_by_filename(self, file_path, current_write_path, meta_data):
"""
:param file_path: file path for send
:param current_write_path: store path index on the storage server
:param meta_data: dictionary, store metadata in it
:return: StorageResponseInfo
* STORAGE_PROTO_CMD_UPLOAD_FILE
# function: upload file to storage server
# request body:
@ 1 byte: store path index on the storage server
@ TRACKER_PROTO_PKG_LEN_SIZE bytes: meta data bytes
@ TRACKER_PROTO_PKG_LEN_SIZE bytes: file size
@ FDFS_FILE_EXT_NAME_MAX_LEN bytes: file ext name, do not include dot (.)
@ meta data bytes: each meta data seperated by \x01,
name and value seperated by \x02
@ file size bytes: file content
# response body: StorageResponseInfo
"""
file_size = os.stat(file_path).st_size
meta_str = self.pack_meta(meta_data)
ext = self.get_ext(file_path)
pkg_len = TRACKER_PROTO_PKG_LEN_SIZE + TRACKER_PROTO_PKG_LEN_SIZE + \
FDFS_FILE_EXT_NAME_MAX_LEN + len(meta_str) + file_size
header = CommandHeader(req_pkg_len=pkg_len, cmd=STORAGE_PROTO_CMD_UPLOAD_FILE)
cmd = Command(pool=self.pool, header=header, fmt="!B %ds %ds %ds %ds" % (TRACKER_PROTO_PKG_LEN_SIZE,
TRACKER_PROTO_PKG_LEN_SIZE,
FDFS_FILE_EXT_NAME_MAX_LEN,
len(meta_str)))
cmd.pack(current_write_path, len(meta_str), file_size, ext, meta_str)
resp, resp_pkg_len = cmd.send_file(file_path)
sr = StorageResponseInfo()
fmt = "!%ds %ds" % (FDFS_GROUP_NAME_MAX_LEN, resp_pkg_len - FDFS_GROUP_NAME_MAX_LEN)
sr.group_name, sr.filename = cmd.unpack(fmt, resp)
return sr
def set_meta(self, file_name, group_name, meta_data, operation_flag=STORAGE_SET_METADATA_FLAG_OVERWRITE):
"""
:param file_name: file name
:param group_name: group name
:param meta_data: dictionary, store metadata in it
:param operation_flag: 'O' for overwrite all old metadata
'M' for merge, insert when the meta item not exist, otherwise update it
:return: none
* STORAGE_PROTO_CMD_SET_METADATA
# function: delete file from storage server
# request body:
@ TRACKER_PROTO_PKG_LEN_SIZE bytes: filename length
@ TRACKER_PROTO_PKG_LEN_SIZE bytes: meta data size
@ 1 bytes: operation flag,
'O' for overwrite all old metadata
'M' for merge, insert when the meta item not exist, otherwise update it
@ FDFS_GROUP_NAME_MAX_LEN bytes: group name
@ filename bytes: filename
@ meta data bytes: each meta data seperated by \x01,
name and value seperated by \x02
# response body: none
"""
file_name_len = len(file_name)
meta_str = self.pack_meta(meta_data)
meta_len = len(meta_str)
pkg_len = TRACKER_PROTO_PKG_LEN_SIZE + TRACKER_PROTO_PKG_LEN_SIZE + 1 + FDFS_GROUP_NAME_MAX_LEN + \
file_name_len + meta_len
header = CommandHeader(req_pkg_len=pkg_len, cmd=STORAGE_PROTO_CMD_SET_METADATA)
cmd_fmt = "!%ds %ds c %ds %ds %ds" %(TRACKER_PROTO_PKG_LEN_SIZE, TRACKER_PROTO_PKG_LEN_SIZE,
FDFS_GROUP_NAME_MAX_LEN, file_name_len, meta_len)
cmd = Command(pool=self.pool, header=header, fmt=cmd_fmt)
cmd.pack(file_name_len, meta_len, operation_flag, group_name, meta_str)
cmd.execute()
def query_fetch_one(self, group_name, file_name):
"""
:param group_name: which group
:param file_name: which file
:return: BasicStorageInfo
* TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH
# function: query which storage server to download the file
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: group name
# response body: BasicStorageInfo
"""
file_name_size = len(file_name)
header = CommandHeader(req_pkg_len=FDFS_GROUP_NAME_MAX_LEN +
file_name_size,
cmd=TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ONE)
cmd = Command(pool=self.pool,
header=header,
fmt="!%ds %ds" %
(FDFS_GROUP_NAME_MAX_LEN, file_name_size))
recv_fmt = '!%ds %ds Q' % (FDFS_GROUP_NAME_MAX_LEN, IP_ADDRESS_SIZE)
cmd.pack(group_name, file_name)
si = BasicStorageInfo()
si.group_name, si.ip_addr, si.port = cmd.fetch_by_fmt(recv_fmt)
return si
def query_store_without_group_all(self):
"""
:return: List<BasicStorageInfo>
* TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL
# function: query which storage server to store file
# request body: none (no body part)
# response body: List<BasicStorageInfo>
"""
header = CommandHeader(
cmd=TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ALL)
cmd = Command(pool=self.pool, header=header)
resp, resp_size = cmd.execute()
server_count = (resp_size - FDFS_GROUP_NAME_MAX_LEN) / (
IP_ADDRESS_SIZE + TRACKER_PROTO_PKG_LEN_SIZE)
recv_fmt = '!%ds %ds %dQ B' % (FDFS_GROUP_NAME_MAX_LEN, server_count *
IP_ADDRESS_SIZE, server_count)
result = cmd.unpack(recv_fmt, resp)
group_name = result[0]
current_write_path = result[-1]
si_list = []
for idx in xrange(server_count):
si = BasicStorageInfo()
si.group_name = group_name
si.current_write_path = current_write_path
si.ip_addr = result[idx + 1]
si.port = result[idx + 1 + server_count]
si_list.append(si)
return si_list
def list_groups(self):
"""
:return: List<GroupInfo>
* TRACKER_PROTO_CMD_SERVER_LIST_GROUP
# function: list all groups
# request body: none
# response body: List<GroupInfo>
"""
header = CommandHeader(cmd=TRACKER_PROTO_CMD_SERVER_LIST_ALL_GROUPS)
cmd = Command(pool=self.pool, header=header)
return cmd.fetch_list(GroupInfo)
def query_store_without_group_one(self):
"""
:return: BasicStorageInfo
* TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE
# function: query storage server for upload, without group name
# request body: none (no body part)
# response body: BasicStorageInfo
"""
header = CommandHeader(
cmd=TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITHOUT_GROUP_ONE)
cmd = Command(pool=self.pool, header=header)
return cmd.fetch_one(BasicStorageInfo)
def query_fetch_all(self, group_name, file_name):
"""
:param group_name: which group
:param file_name: which file
:return: List<BasicStorageInfo>
* TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL
# function: query all storage servers to download the file
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: group name
@ filename bytes: filename
# response body: List<BasicStorageInfo>
"""
file_name_size = len(file_name)
header = CommandHeader(req_pkg_len=FDFS_GROUP_NAME_MAX_LEN +
file_name_size,
cmd=TRACKER_PROTO_CMD_SERVICE_QUERY_FETCH_ALL)
cmd = Command(pool=self.pool,
header=header,
fmt="!%ds %ds" %
(FDFS_GROUP_NAME_MAX_LEN, file_name_size))
cmd.pack(group_name, file_name)
resp, resp_size = cmd.execute()
server_count = (resp_size - FDFS_GROUP_NAME_MAX_LEN - 1 -
TRACKER_PROTO_PKG_LEN_SIZE -
IP_ADDRESS_SIZE) / IP_ADDRESS_SIZE
recv_fmt = '!%ds %ds Q %ds' % (FDFS_GROUP_NAME_MAX_LEN,
IP_ADDRESS_SIZE,
server_count * IP_ADDRESS_SIZE)
result = cmd.unpack(recv_fmt, resp)
group_name = result[0]
server_port = result[2]
si_list = []
si = BasicStorageInfo()
si.group_name = group_name
si.ip_addr = result[1]
si.port = server_port
si_list.append(si)
for idx in xrange(server_count):
si = BasicStorageInfo()
si.group_name = group_name
si.ip_addr = result[idx + 3]
si.port = server_port
si_list.append(si)
return si_list
def list_one_group(self, group_name):
"""
:param: group_name: which group
:return: GroupInfo
* TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP
# function: get one group info
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: the group name to query
# response body: GroupInfo
"""
header = CommandHeader(req_pkg_len=FDFS_GROUP_NAME_MAX_LEN,
cmd=TRACKER_PROTO_CMD_SERVER_LIST_ONE_GROUP)
cmd = Command(pool=self.pool,
header=header,
fmt='!%ds' % FDFS_GROUP_NAME_MAX_LEN)
cmd.pack(group_name)
return cmd.fetch_one(GroupInfo)
def delete_file(self, group_name, file_name):
"""
:param group_name:
:param file_name:
:return: none
* STORAGE_PROTO_CMD_DELETE_FILE
# function: delete file from storage server
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: group name
@ filename bytes: filename
# response body: none
"""
file_name_len = len(file_name)
header = CommandHeader(req_pkg_len=FDFS_GROUP_NAME_MAX_LEN + file_name_len,
cmd=STORAGE_PROTO_CMD_DELETE_FILE)
cmd = Command(pool=self.pool, header=header, fmt="! %ds %ds" % (FDFS_GROUP_NAME_MAX_LEN, file_name_len))
cmd.pack(group_name, file_name)
cmd.execute()
def query_store_with_group_one(self, group_name):
"""
:param: group_name: which group
:return: BasicStorageInfo
* TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE
# function: query storage server for upload, with group name
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: the group name to
# response body: BasicStorageInfo
"""
header = CommandHeader(
req_pkg_len=FDFS_GROUP_NAME_MAX_LEN,
cmd=TRACKER_PROTO_CMD_SERVICE_QUERY_STORE_WITH_GROUP_ONE)
cmd = Command(pool=self.pool,
header=header,
fmt='!%ds' % FDFS_GROUP_NAME_MAX_LEN)
cmd.pack(group_name)
return cmd.fetch_one(BasicStorageInfo)
def list_servers(self, group_name, storage_ip=None):
"""
:param: group_name: which group
:param: storage_ip: which storage servers
:return: List<StorageInfo>
* TRACKER_PROTO_CMD_SERVER_LIST_STORAGE
# function: list storage servers of a group
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: the group name to
@ IP_ADDRESS_SIZE bytes: this storage server ip address
# response body: List<StorageInfo>
"""
ip_len = IP_ADDRESS_SIZE if storage_ip else 0
header = CommandHeader(req_pkg_len=FDFS_GROUP_NAME_MAX_LEN + ip_len,
cmd=TRACKER_PROTO_CMD_SERVER_LIST_STORAGE)
cmd = Command(pool=self.pool,
header=header,
fmt="!%ds %ds" % (FDFS_GROUP_NAME_MAX_LEN, ip_len))
cmd.pack(group_name, storage_ip or "")
return cmd.fetch_list(StorageInfo)
def get_meta(self, group_name, file_name):
"""
:param group_name: group name
:param file_name: file name
:return: meta data, dictionary, store metadata in it
* STORAGE_PROTO_CMD_GET_METADATA
# function: get metat data from storage server
# request body:
@ FDFS_GROUP_NAME_MAX_LEN bytes: group name
@ filename bytes: filename
# response body
@ meta data buff, each meta data seperated by \x01, name and value seperated by \x02
"""
header = CommandHeader(req_pkg_len=FDFS_GROUP_NAME_MAX_LEN + len(file_name), cmd=STORAGE_PROTO_CMD_GET_METADATA)
cmd = Command(pool=self.pool, header=header, fmt="!%ds %ds" % (FDFS_GROUP_NAME_MAX_LEN, len(file_name)))
cmd.pack(group_name, file_name)
resp, resp_size = cmd.execute()
meta_data = {}
for item in resp.split(FDFS_RECORD_SEPARATOR):
k, v = item.split(FDFS_FIELD_SEPARATOR)
meta_data[k] = v
return meta_data