def test_client_delete():
args_filepath = input("request(json absolute filepath): ")
with open(args_filepath) as fin:
query_body_unpack = json.load(fin)
print(json.dumps(query_body_unpack, indent=4, ensure_ascii=False))
site_id = query_body_unpack.get("site_id")
app_id = query_body_unpack.get("app_id")
timestamp = query_body_unpack.get("timestamp")
file_name = query_body_unpack.get("file_name").encode("utf-8")
file_md5 = query_body_unpack.get("file_md5").encode("utf-8")
# body
t_body = Body()
t_body.site_id = int(site_id)
t_body.app_id = int(app_id)
t_body.timestamp = int(timestamp)
t_body.file_name = file_name
t_body.file_md5 = file_md5
body_pack = t_body.pack()
body_size = len(body_pack)
# meta
del query_body_unpack["site_id"]
del query_body_unpack["app_id"]
del query_body_unpack["timestamp"]
del query_body_unpack["file_name"]
del query_body_unpack["file_md5"]
query_body_pack = json.dumps(query_body_unpack).encode("utf-8")
query_body_size = len(query_body_pack)
# header
t_header = Header()
t_header.command = config.Constant.CLIENT_DEL
head_size = struct.calcsize(config.Constant.FMT_COMMON_HEAD)
t_header.total_size = head_size + body_size + query_body_size
header_pack = t_header.pack()
# packet
pkt = header_pack + body_pack + query_body_pack
# send request
host = input("host: ")
port = input("port: ")
clisock = utils.create_client_socket(host, int(port))
clisock.sendall(pkt)
# recv response
recvlen, header_pack = utils.attempt_recvall(clisock,
config.Constant.HEAD_LENGTH)
if recvlen == config.Constant.HEAD_LENGTH:
header_unpack = struct.unpack(config.Constant.FMT_COMMON_HEAD,
header_pack)
ack_code = header_unpack[10]
if int(ack_code) == 200:
print("test_client_delete: OK")
else:
print("test_client_delete: FAIL (ack={0})".format(ack_code))
else:
print("test_client_delete: FAIL (recv uncompleted message)")
def start_to_upload(self):
file_nums = self.work_queue.qsize()
sgw_sock = utils.create_client_socket(config_info.sgw_host,
config_info.sgw_port)
while True:
if self.work_queue.qsize() == 0:
print("upload task finished.")
# 通知mds study上传完成
break
source_path, file_block_info, block_id_info = self.work_queue.get()
_, _, file_size = file_block_info
if file_size > self.MD5_BLOCK_SIZE:
file_obj = open(source_path, 'rb')
else:
with open(source_path, 'rb') as fr:
file_obj = io.BytesIO(fr.read())
cnt = 0
while cnt <= 3:
cnt += 1
self.handle_upload(sgw_sock, file_obj, file_block_info,
source_path)
# if Upload(source_path).upload_file(sgw_sock, file_obj, file_block_info):
break
else:
print("upload {} failed.".format(source_path))
file_obj.close()
sgw_sock.close()
return file_nums
def upload_single_task():
while True:
mds_host = config_info.host
mds_tcp_port = config_info.mds_port
clisock = utils.create_client_socket(mds_host, int(mds_tcp_port))
cli_obj = HandleCommon(config_info.site_id)
cli_obj.msg_headers.command = 0x0010
cli_obj.msg_headers.trans_id = create_trans_id()
time_a = time.strptime('2000-01-01 00:00:01', "%Y-%m-%d %H:%M:%S")
time_send = int(time.mktime(time_a))
query_param = {
'timestamp_before': time_send,
'study_state': 0,
'nr_study': 5,
'study_id': '',
'nr_files': 0
}
# header
body = json.dumps(query_param)
msg = cli_obj.pack_data(body.encode())
clisock.sendall(msg)
_, _, _, body = utils.get_msg(clisock, config.Constant.HEAD_LENGTH,
config.Constant.FMT_COMMON_HEAD)
upload_study_list = json.loads(body.decode())['study_list']
print("upload_study_list: {}".format(upload_study_list))
study_absolute_path_l = []
for upload_i in upload_study_list:
absolute_path = os.path.join(config_info.upload_path,
study_id_to_path(upload_i), upload_i)
study_absolute_path_l.append(absolute_path)
# 构建消息体
upload_obj = Upload(absolute_path)
upload_obj.queue_create()
counts = upload_obj.start_to_upload()
query_param = {
'timestamp_before': 0,
'study_state': 0,
'nr_study': 0,
'study_id': upload_i,
'nr_files': counts
}
cli_obj = HandleCommon(config_info.site_id)
cli_obj.msg_headers.command = 0x0010
cli_obj.msg_headers.trans_id = create_trans_id()
body = json.dumps(query_param)
msg = cli_obj.pack_data(body.encode())
clisock.sendall(msg)
clisock.close()
if not upload_study_list:
break
def get_file_list_from_sgw(folder):
handle_common = HandleCommon(config_info.site_id)
get_file_list_trans_id = create_trans_id()
sgw_ip = socket.ntohl(
struct.unpack("I", socket.inet_aton(config_info.sgw_host))[0])
sgw_port = int(config_info.sgw_port)
handle_common.task_info.sgw_ip = sgw_ip
handle_common.task_info.sgw_port = sgw_port
handle_common.task_info.proxy_ip = sgw_ip
handle_common.task_info.proxy_port = sgw_port
handle_common.task_info.file_name = folder.encode()
handle_common.task_info.metadata = b''
sequence = 0
handle_common.msg_headers.src_type = 1
handle_common.msg_headers.dst_type = 3
handle_common.msg_headers.dst_id = handle_common.task_info.proxy_id
handle_common.msg_headers.command = 0x2000F
handle_common.msg_headers.trans_id = get_file_list_trans_id
handle_common.msg_headers.sequence = sequence
sgw_sock = utils.create_client_socket(config_info.sgw_host, sgw_port)
send_data = handle_common.pack_data()
sgw_sock.sendall(send_data)
head_msg = recv_cycle(sgw_sock, sizeof(c_longlong))
msg_length, = struct.unpack(b'>q', head_msg)
body_msg = recv_cycle(sgw_sock, msg_length - sizeof(c_longlong))
sgw_sock.close()
recv_bytes = head_msg + body_msg
if not recv_bytes:
print('sgw socket receive result is %s' % recv_bytes)
return {}
file_info_bytes = recv_bytes[sizeof(c_longlong):]
file_num, = struct.unpack(b'>I', file_info_bytes[0:sizeof(c_uint)])
file_list_bytes = file_info_bytes[sizeof(c_uint):]
file_info_dict = {}
for _ in range(file_num):
file_path_len, = struct.unpack(b'>H',
file_list_bytes[0:sizeof(c_ushort)])
file_size_cursor = sizeof(c_ushort) + file_path_len
file_path = file_list_bytes[sizeof(c_ushort):file_size_cursor]
file_size, = struct.unpack(
b'>q', file_list_bytes[file_size_cursor:file_size_cursor +
sizeof(c_longlong)])
file_info_dict[file_path] = file_size
file_list_bytes = file_list_bytes[file_size_cursor +
sizeof(c_longlong):]
return file_num, file_info_dict
def test_migration_start():
args_filepath = input("request(json absolute filepath): ")
with open(args_filepath) as fin:
migoption = json.load(fin)
print(json.dumps(migoption, indent=4, ensure_ascii=False))
old_sgw_ip = migoption.get("old_sgw_ip").encode("utf-8")
old_sgw_port = migoption.get("old_sgw_port")
new_sgw_ip = migoption.get("new_sgw_ip").encode("utf-8")
new_sgw_port = migoption.get("new_sgw_port")
old_mds_ip = migoption.get("old_mds_ip").encode("utf-8")
old_mds_port = migoption.get("old_mds_port")
new_mds_ip = migoption.get("new_mds_ip").encode("utf-8")
new_mds_port = migoption.get("new_mds_port")
# payload
fmt_migoption = "!64sH64sH64sH64sH"
migoption_unpack = (old_sgw_ip, old_sgw_port, new_sgw_ip, new_sgw_port,
old_mds_ip, old_mds_port, new_mds_ip, new_mds_port)
migoption_pack = struct.pack(fmt_migoption, *migoption_unpack)
print("migoption_pack size: ", struct.calcsize(fmt_migoption))
print("migoption_pack len: ", len(migoption_pack))
# Header
t_header = Header()
t_header.total_size = 64 + len(migoption_pack)
t_header.command = 0x00030001
header_pack = t_header.pack()
# packet
pkt = header_pack + migoption_pack
# send request to sgw
clisock = utils.create_client_socket(old_sgw_ip, old_sgw_port)
clisock.sendall(pkt)
# recv response
recvlen, header_pack = utils.attempt_recvall(clisock,
config.Constant.HEAD_LENGTH)
if recvlen == config.Constant.HEAD_LENGTH:
header_unpack = struct.unpack(config.Constant.FMT_COMMON_HEAD,
header_pack)
ack_code = header_unpack[10]
if int(ack_code) == 200:
print("test_migration_start: OK")
else:
print("test_migration_start: FAIL (ack={0})".format(ack_code))
else:
print("test_migration_start: FAIL (recv uncompleted message)")
def download(save_path, sgw_path, file_size):
handle_common = HandleCommon(config_info.site_id)
download_req_trans_id = create_trans_id()
handle_common.task_info = TaskInfo()
sgw_ip = socket.ntohl(
struct.unpack("I", socket.inet_aton(config_info.sgw_host))[0])
sgw_port = config_info.sgw_port
handle_common.task_info.sgw_ip = sgw_ip
handle_common.task_info.sgw_port = sgw_port
handle_common.task_info.proxy_ip = sgw_ip
handle_common.task_info.proxy_port = sgw_port
file_path_in_sgw = sgw_path
handle_common.task_info.file_name = file_path_in_sgw
handle_common.msg_headers.src_type = 1
handle_common.msg_headers.dst_type = 3
handle_common.msg_headers.dst_id = handle_common.task_info.proxy_id
handle_common.msg_headers.trans_id = download_req_trans_id
handle_common.msg_headers.sequence = 0
# sgw_sock = handle_common.connect_sgw()
sgw_sock = utils.create_client_socket(config_info.sgw_host, int(sgw_port))
with open(save_path, 'auto_build+b') as fp:
handle_common.msg_headers.command = 0x20011
handle_common.msg_headers.total = file_size
handle_common.msg_headers.offset = 0
handle_common.msg_headers.count = file_size
sgw_sock.sendall(handle_common.pack_data())
head_msg = recv_cycle(sgw_sock, sizeof(c_longlong))
msg_length, = struct.unpack(b'>q', head_msg)
body_msg = recv_cycle(sgw_sock, msg_length - sizeof(c_longlong))
recv_bytes = head_msg + body_msg
if recv_bytes:
file_bytes = recv_bytes[sizeof(c_longlong):]
if file_bytes:
fp.write(file_bytes)
download_res = True
else:
print('下载文件为空')
download_res = False
else:
print('下载文件出错')
download_res = False
sgw_sock.close()
return download_res
def move_back_notify(send_msg):
handle_common = HandleCommon(config_info.site_id)
mds_sock = utils.create_client_socket(config_info.host,
config_info.mds_port)
trans_id = create_trans_id()
handle_common.msg_headers.command = 0x0014
handle_common.msg_headers.trans_id = trans_id
body = json.dumps(send_msg)
msg = handle_common.pack_data(body.encode())
mds_sock.sendall(msg)
header_structure = MsgHeaders()
header_len = sizeof(header_structure)
head_msg = recv_cycle(mds_sock, header_len)
msg_length, = struct.unpack(b'>I', head_msg[0:sizeof(c_uint32)])
if msg_length - header_len > 0:
body_msg = recv_cycle(mds_sock, msg_length - header_len)
mds_sock.close()
memmove(addressof(header_structure), head_msg, header_len)
if header_structure.ack_code == 200:
print("notice MOVE_BACK_NOTIFY to metadata success.")
elif header_structure.ack_code == 404:
print("notice MOVE_BACK_NOTIFY to metadata failed.")
def test_client_query_file_number():
args_filepath = config_info.study_upload_conf
# args_filepath = input("request(json absolute filepath): ")
with open(args_filepath) as fin:
query_body_unpack = json.load(fin)
print(json.dumps(query_body_unpack, indent=4, ensure_ascii=False))
query_body_pack = json.dumps(query_body_unpack).encode("utf-8")
query_body_size = len(query_body_pack)
head_size = struct.calcsize(config.Constant.FMT_COMMON_HEAD)
total_size = head_size + query_body_size
# header
t_header = Header()
t_header.total_size = total_size
t_header.command = config.Constant.CLIENT_QUERY_NUM
header_pack = t_header.pack()
# packet
pkt = header_pack + query_body_pack
# send request
host = input("host: ")
port = input("port: ")
clisock = utils.create_client_socket(host, int(port))
clisock.sendall(pkt)
# recv response
recvlen, header_pack = utils.attempt_recvall(clisock,
config.Constant.HEAD_LENGTH)
if recvlen == config.Constant.HEAD_LENGTH:
header_unpack = struct.unpack(config.Constant.FMT_COMMON_HEAD,
header_pack)
print("nr_files:", header_unpack[11])
print("test_client_query_file_num: OK")
else:
print("test_client_query_file_num: FAIL (recv uncompleted message)")
def __init__(self, node):
Thread.__init__(self)
self._node = node
self._client_sockets = [utils.create_client_socket() for i in
xrange(config.NUM_NODE)]
def test_client_query_file_data():
args_filepath = input("request(json absolute filepath): ")
with open(args_filepath) as fin:
query_body_unpack = json.load(fin)
print(json.dumps(query_body_unpack, indent=4, ensure_ascii=False))
query_body_pack = json.dumps(query_body_unpack).encode('utf-8')
query_body_size = len(query_body_pack)
head_size = struct.calcsize(config.Constant.FMT_COMMON_HEAD)
total_size = head_size + query_body_size
# header
t_header = Header()
t_header.total_size = total_size
t_header.command = config.Constant.CLIENT_QUERY_DATA
header_pack = t_header.pack()
# packet
pkt = header_pack + query_body_pack
# send request
host = input("host: ")
port = input("port: ")
clisock = utils.create_client_socket(host, int(port))
clisock.sendall(pkt)
def parse(entry):
fixlen = struct.calcsize(config.Constant.FMT_TASKINFO_FIXED)
body_pack = entry[:fixlen]
body_unpack = struct.unpack(config.Constant.FMT_TASKINFO_FIXED,
body_pack)
metalen = body_unpack[15]
return (fixlen, metalen)
def output(meta_unpack):
print("user_id:", meta_unpack.get("user_id"))
print("customer_id:", meta_unpack.get("customer_id"))
print("file_path:", meta_unpack.get("file_path"))
print("file_name:", meta_unpack.get("file_name"))
print("file_md5:", meta_unpack.get("file_md5"))
print("thumb_md5", meta_unpack.get("thumb_md5"))
print("nr_blocks:", meta_unpack.get("nr_blocks"))
blocks = meta_unpack.get("blocks")
for block_id, block_md5 in enumerate(blocks):
print("block: {0} {1} {2}".format(block_id, block_md5,
meta_unpack.get(block_md5)))
print("auto_examine:", meta_unpack.get("auto_examine"))
print("text:", meta_unpack.get("text"))
examine_details = meta_unpack.get("examine_details")
if examine_details:
for detail in examine_details:
print("word: {0}, time: {1}".format(detail.get("word"),
detail.get("time")))
def parse_and_output(pkt):
while len(pkt) > 0:
fixlen, metalen = parse(pkt)
meta_pack = pkt[fixlen:fixlen + metalen]
meta_unpack = json.loads(meta_pack.decode('utf-8'))
output(meta_unpack)
pkt = pkt[fixlen + metalen:]
# recv response
hdrlen, header_unpack, bodylen, body_pack = utils.get_msg(
clisock, config.Constant.HEAD_LENGTH, config.Constant.FMT_COMMON_HEAD)
if header_unpack:
ack_code = header_unpack[10]
if int(ack_code) == 200:
if body_pack:
parse_and_output(body_pack)
print("test_client_query_data: OK")
else:
print("test_client_query_data: OK (no file metadata)")
else:
print(
"test_client_query_file_data: FAIL (ack={0})".format(ack_code))
else:
print("test_client_query_file_data: FAIL (uncompleted header)")
