def run(self):
try:
packlen = 32
count = 0
buf = self.sock.recv(packlen)
while len(buf) < packlen and count < 10:
buf1 = self.sock.recv(packlen-len(buf), socket.MSG_DONTWAIT)
buf += buf1
time.sleep(1)
count += 1
if len(buf) < packlen:
common._logger.info("recved less than %d bytes in 10 seconds:%s, exit." % (packlen, buf))
return
else:
pass
self.addr = DNS._resolve(self.ip, 0)
global machine_lock
machine_lock.acquire()
try:
if buf[:4] == 'live':
repo=buf.split(' ')[0][4:]
ip=buf.split(' ')[1]
self.live(repo, ip)
elif buf[:4] == 'dead':
pass
else:
common._logger.info("error header in scheduler importer, close socket")
finally:
machine_lock.release()
except:
common.print_exc_plus()
finally:
self.sock.close()
def list_slice(table):
"ret list: [Slice]"
slice_and_state_count = [[], {},
{}] # [[slice_list],state->num,state->slice_list]
global g_slice_states
for state in g_slice_states:
slice_and_state_count[1][state] = 0
slice_and_state_count[2][state] = []
try:
shell_out = common.run_shell(
'echo "list slice on table %s;" | ./Cli -s' % table)
output_lines = shell_out.split('\n')
n = len(output_lines)
if len(output_lines) > 7:
status = output_lines[7].split('|')[3].strip()
if status != 'ok':
pass # TODO
else:
for line_idx in range(13, n, 2):
stmp = output_lines[line_idx]
if stmp != '':
slice = BailingSlice.Slice(stmp)
slice_and_state_count[1][slice.state] += 1
slice_and_state_count[2][slice.state].append(slice)
slice_and_state_count[0].append(slice)
return slice_and_state_count
except:
common.print_exc_plus()
def run(self):
try:
count = 0
buf = self.sock.recv(8)
while len(buf) < 8 and count < 10:
buf1 = self.sock.recv(8 - len(buf), socket.MSG_DONTWAIT)
buf += buf1
time.sleep(1)
count += 1
if len(buf) < 8:
common._logger.warn("recved less than 8 bytes:%s, exit." % buf)
return
else:
common._logger.info("recved 8 bytes:%s" % buf)
if buf[:3] == 'get':
self.get_undo_slice()
elif buf[:3] == 'end':
n = int(buf[3:])
self.finish_slice(n)
elif buf[:3] == 'err':
n = int(buf[3:])
self.reschedule_slice(n)
else:
common._logger.info(
"error header in scheduler importer, close socket")
except:
common.print_exc_plus()
finally:
self.sock.close()
def __init__(self):
Thread.__init__(self)
global schedule_data
schedule_data.lock.acquire()
try:
start = Settings.slice_start
end = Settings.slice_end
self.slices_and_stateCount = Cli.list_slice(Settings.import_repo)
slices = self.slices_and_stateCount[0]
slice_ids = []
slice_machines = {}
# get specified_slices info
specified_slices = set()
specified_flag = False
f = open('import_specified_slices', 'r')
for line in f:
specified_slices.add(int(line))
if len(specified_slices) > 0:
specified_flag = True
for s in slices:
if start <= s.slice_id <= end:
if specified_flag:
if s.slice_id not in specified_slices:
continue
slice_ids.append(s.slice_id)
schedule_data.slice_done_flag[
s.slice_id] = SliceProcessState.TODO
ret_list = Cli.show_slice(Settings.import_repo, s.slice_id)
slice_machines[s.slice_id] = []
for unit in ret_list[1]:
slice_machines[s.slice_id].append(unit.node)
schedule_data.slice_ids = slice_ids
schedule_data.slice2machines = slice_machines
machines = Cli.list_server()
for m in machines:
schedule_data.machine_list[m.addr.ip + ':' +
str(m.addr.port)] = 0
# fp = open('slice_done', 'r')
# for line in fp:
# n, state = line.strip().split(':')
# if state == '0':
# state = SliceProcessState.TODO
# elif state == '1':
# state = SliceProcessState.IMPORTING
# elif state == '2':
# state = SliceProcessState.IMPORTED
# else:
# raise Exception('Error at parsing file slice_done.')
# schedule_data.slice_done_flag[int(n)] = state
except:
common.print_exc_plus()
finally:
schedule_data.lock.release()
def run(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('', Settings.schedule_port))
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.listen(5)
while True:
try:
client_socket, addr = s.accept()
common._logger.info("get accepted at (ip:%s, port:%s)" %
(addr[0], addr[1]))
ScheduleSlice(client_socket).start()
except:
common.print_exc_plus()
def show_slice(table, slice_no):
"ret_list: [Slice, unit_list, snapshot_list]"
ret_list = []
try:
shell_out = common.run_shell(
'echo "show slice %d on table %s;" | ./Cli -s' %
(int(slice_no), table))
output_lines = shell_out.split('\n')
n = len(output_lines)
if len(output_lines) > 7:
status = output_lines[7].split('|')[3].strip()
if status != 'ok':
pass # TODO
else:
sli = BailingSlice.Slice(output_lines[13])
ret_list.append(sli)
# rep_base = snapshot_base = 0
for idx, line in enumerate(output_lines):
line = line.strip()
if line == 'Replication List:':
rep_base = idx
elif line == 'Snapshot List:' or line == 'Snapshot list:':
snapshot_base = idx
rep_list = []
for line_idx in range(rep_base + 4, snapshot_base, 2):
stmp = output_lines[line_idx]
if stmp != '':
rep_list.append(stmp.split('|')[1].strip())
unit_list = []
for line_idx in range(19, rep_base, 2):
stmp = output_lines[line_idx]
if stmp != '':
unit = BailingSlice.Unit(stmp)
try:
unit.rep_order = rep_list.index(unit.repl_addr)
except:
unit.rep_order = 999
unit_list.append(unit)
ret_list.append(unit_list)
snapshot_list = []
for line_idx in range(snapshot_base + 4, len(output_lines), 2):
stmp = output_lines[line_idx]
if stmp != '':
snapshot_list.append(BailingSlice.Snapshot(stmp))
ret_list.append(snapshot_list)
return ret_list
except:
common.print_exc_plus()
def list_server():
machine_list = []
try:
shell_out = common.run_shell('echo "list server;" | ./Cli -s')
output_lines = shell_out.split('\n')
n = len(output_lines)
if len(output_lines) > 7:
status = output_lines[7].split('|')[3].strip()
if status != 'ok':
pass # TODO
else:
for line_idx in range(13, n, 2):
if output_lines[line_idx] != '':
machine_list.append(
MachineManager.Machine(output_lines[line_idx]))
return machine_list
except:
common.print_exc_plus()
def list_table():
"ret list: [Repository]"
table_list = []
try:
shell_out = common.run_shell('echo "list table;" | ./Cli -s')
output_lines = shell_out.split('\n')
n = len(output_lines)
if len(output_lines) > 7:
status = output_lines[7].split('|')[3].strip()
if status != 'ok':
pass # TODO
else:
for line_idx in range(13, n, 2):
stmp = output_lines[line_idx]
if stmp != '':
table_list.append(BailingRepository.Repository(stmp))
return table_list
except:
common.print_exc_plus()
def show_snapshot(table, ss):
ret_list = []
try:
shell_out = common.run_shell(
'echo "show snapshot %s on table %s;" | ./Cli -s' % (ss, table))
output_lines = shell_out.split('\n')
n = len(output_lines)
if len(output_lines) > 7:
status = output_lines[7].split('|')[3].strip()
if status != 'ok':
pass # TODO
else:
for line_idx in range(13, len(output_lines), 2):
stmp = output_lines[line_idx]
if stmp != '':
ret_list.append(stmp.split('|')[1:-1])
return ret_list
except:
common.print_exc_plus()