def start_load(self, master, cqid, node, database, tag, db_col_sep, opt):
logging("start to load")
table = opt.table
files = opt.files
fashion = opt.fashion
key = opt.key
key_pos = opt.key_pos
row_sep = opt.row_sep
col_sep = opt.col_sep
dload_client.dload_client().load_internal(master, cqid, node, database, table,
files, 0, fashion, key, key_pos,
db_col_sep, row_sep, col_sep,
opt.replace, "0",
self.defaultconf[conf.LOG_DIR])
logging("%s: FINISH" % (self.start_load.__name__))
def handle_read(self, event):
message = event.data[10:]
m = message.split(conf.SEP_IN_MSG)
try:
if m[0] == conf.JOB_ARGUMENT:
self.parse_args(m[1])
ParaLiteLog.info("parse arguments: FINISH")
elif m[0] == conf.JOB:
ParaLiteLog.debug("MESSAGE: %s" % message)
self.cur_jobid = m[1]
elif m[0] == conf.DATA:
data_id = string.strip(m[1][0:2])
data = m[1][2:]
self.source_data.append(data)
# sort data
if not self.is_data_ready(self.source_data, self.num_of_children):
return
ParaLiteLog.debug("****SORT DATA****: start")
s = 0
for data in self.source_data:
s += len(data)
ParaLiteLog.debug("source data size: %s" % s)
s_time = time.time()
rs_type, rs, t_size = self.sort(self.source_data)
del self.source_data
ParaLiteLog.debug("****SORT DATA****: finish")
if rs_type is None:
self.send_status_to_master(self.cur_jobid, conf.PENDING)
return
self.total_size += t_size
self.source_data = {}
# store the result of one job to the final result
for i in range(len(rs)):
if i not in self.result:
self.result[i] = [rs[i]]
else:
self.result[i].append(rs[i])
if rs_type != conf.MULTI_FILE:
# check if the whole data exceeds the LIMITATION
if self.total_size > self.MAX_SIZE:
self.write_data_to_disk()
self.result_type = conf.MULTI_FILE
e_time = time.time()
self.total_time += e_time - s_time
self.send_status_to_master(self.cur_jobid, conf.PENDING)
elif m[0] == conf.JOB_END:
ParaLiteLog.debug("MESSAGE: %s" % message)
# all jobs are finished
self.send_rs_info_to_master(self.total_size, self.total_time)
# distribute data
if self.dest == conf.DATA_TO_ONE_CLIENT:
ParaLiteLog.debug("dest = %s" % self.dest)
self.distribute_data()
self.send_status_to_master(self.cur_jobid, conf.ACK)
self.is_running = False
elif self.dest == conf.DATA_TO_DB:
self.distribute_data()
elif m[0] == conf.DATA_PERSIST:
# if the data is requried to be persisted or not
if m[1] == conf.CHECKPOINT:
self.write_data_to_disk()
elif m[0] == conf.DLOAD_REPLY:
sep = conf.SEP_IN_MSG
reply = sep.join(m[1:])
ParaLiteLog.info("receive the information from the master")
ParaLiteLog.debug(reply)
if len(self.data.getvalue()) != 0:
dload_client.dload_client().load_internal_buffer(
reply,
self.dest_table,
self.data,
self.fashion,
self.hash_key,
self.hash_key_pos,
self.db_col_sep,
self.db_row_sep,
self.db_col_sep,
False,
"0",
self.log_dir,
)
# send END_TAG to the master
client_id = "0"
msg = sep.join([conf.REQ, conf.END_TAG, gethostname(), client_id])
so_master = socket(AF_INET, SOCK_STREAM)
so_master.connect((self.master_name, self.master_port))
so_master.send("%10s%s" % (len(msg), msg))
so_master.close()
ParaLiteLog.debug("sending to master: %s" % (conf.END_TAG))
ParaLiteLog.debug("----- dload client finish -------")
elif message == conf.DLOAD_END_TAG:
ParaLiteLog.debug("---------import finish---------")
self.send_status_to_master(" ".join(self.cur_jobid), conf.ACK)
self.is_running = False
elif m[0] == conf.EXIT:
self.is_running = False
elif m[0] == conf.NODE_FAIL:
ParaLiteLog.debug("MESSAGE: %s" % message)
# message --> NODE_FAIL:FAILED_NODE:REPLICA_NODE
failed_node, replica_node = m[1:3]
self.failed_node.append(failed_node)
if replica_node != "" and replica_node == gethostname():
# load replica data for the failed node
self.recovery_data(self.replica_result, replica_node)
ParaLiteLog.debug("Finish to handle node failure message")
except Exception, e:
es(traceback.format_exc())
ParaLiteLog.info(traceback.format_exc())
self.is_running = False
self.no_error = False
else:
addr = (destnode, self.p_node[destnode][0])
t = AF_INET
self.send_data_to_node(msg, t, addr)
ParaLiteLog.debug(
"send data susscufully %s %s --> %s" % (
self.opid, gethostname(), destnode))
elif m[0] == conf.DLOAD_REPLY:
reply = sep.join(m[1:])
ParaLiteLog.info("receive the information from the master")
ParaLiteLog.debug(reply)
if len(self.data.getvalue()) != 0:
dload_client.dload_client().load_internal_buffer(
reply, self.dest_table, self.data, self.fashion,
self.hash_key, self.hash_key_pos, self.db_col_sep,
self.db_row_sep, self.db_col_sep, False, "0", self.log_dir)
# send END_TAG to the master
client_id = "0"
msg = sep.join([conf.REQ, conf.END_TAG, gethostname(), client_id])
so_master = socket(AF_INET, SOCK_STREAM)
so_master.connect((self.master_name, self.master_port))
so_master.send("%10s%s" % (len(msg), msg))
so_master.close()
ParaLiteLog.debug("sending to master: %s" % (conf.END_TAG))
ParaLiteLog.debug("----- dload client finish -------")
elif message == conf.DLOAD_END_TAG:
ParaLiteLog.debug("---------import finish---------")
self.send_status_to_master(" ".join(self.job_data), conf.ACK)
def handle_read(self, event):
message = event.data[10:]
sep = conf.SEP_IN_MSG
m = message.split(sep)
try:
if m[0] == conf.DATA_END:
ParaLiteLog.debug("MESSAGE: %s" % message)
# all data is dipatched to the parent nodes
self.send_status_to_master(" ".join(self.job_data), conf.ACK)
ParaLiteLog.debug("notify ACK to master")
self.is_running = False
elif message == conf.END_TAG:
ParaLiteLog.debug("MESSAGE: %s" % message)
self.send_status_to_master(" ".join(self.job_data), conf.ACK)
self.is_running = False
elif message == conf.DLOAD_END_TAG:
ParaLiteLog.debug("---------import finish---------")
self.send_status_to_master(" ".join(self.job_data), conf.ACK)
self.is_running = False
elif message == conf.EXIT:
ParaLiteLog.debug("MESSAGE: %s" % message)
self.is_running = False
elif m[0] == conf.JOB_ARGUMENT:
self.parse_args(m[1])
ParaLiteLog.info("parse arguments: FINISH")
# init the persisted result data
if self.is_checkpoint is not None and self.is_checkpoint == conf.CHECKPOINT:
ParaLiteLog.debug("recovery data: START")
# this is a recovery operator
self.recovery_data(self.result, gethostname())
ParaLiteLog.debug("recovery data: FINISH")
self.send_rs_info_to_master(0, 0)
else:
# delete all temporary files for this operator
os.system("rm -f %s/%s_%s" % (self.temp_dir, "sql", self.opid))
###############################
# scanner = threading.Thread(target=self.scan_process_queue, args=(self.process_queue, ))
# scanner.setDaemon(True)
# scanner.start()
# self.threads.append(scanner)
##########################
elif m[0] == conf.JOB:
self.ex_s_time = time.time()
self.ex_w_time = 0
ParaLiteLog.debug("MESSAGE: %s" % message)
s_time = time.time()
jobid = m[1]
target_db = m[2].split()
exp = self.expression
ParaLiteLog.debug("*****JOB %s******:start" % jobid)
# FAULT TOLERANCE:
if jobid in self.job_data:
# this is a failed job, we should first delete the old result value
if self.dest == conf.DATA_TO_ANO_OP and self.partition_num > 1:
for partid in self.result:
pos = self.job_list.index(jobid)
self.result[partid][pos] = ""
else:
self.result[jobid] = ""
if exp.lower().startswith("select"):
"""
selection task: (1), execute sql (2), notify the result to
the master (3), wait for the DATA_PERSIST message from the
master (4), persist data if so (5), notify ACK to the master
"""
ParaLiteLog.info("proc_select: START")
st_time = time.time()
rs_type, rs, t_size = self.proc_select(jobid, exp, target_db)
et_time = time.time()
ParaLiteLog.debug("Job %s cost time %s second" % (jobid, (et_time - st_time)))
# FAULT TOLERANCE:
if jobid in self.job_data:
# this is a failed job
if self.dest == conf.DATA_TO_ANO_OP and self.partition_num > 1:
for partid in self.result:
pos = self.job_list.index(jobid)
self.result[partid][pos] = rs[partid]
else:
self.result[jobid] = rs
self.send_status_to_master(jobid, conf.PENDING)
return
self.job_data[jobid] = t_size
self.job_list.append(jobid)
self.total_size += t_size
# store the result of one job to the final result
if len(rs) == 1:
if self.dest == conf.DATA_TO_ANO_OP:
# dest is AGGR op or ORDER op, use 0 as the key
if 0 not in self.result:
self.result[0] = rs
else:
self.result[0].append(rs[0])
if self.is_checkpoint == 1:
self.write_data_to_disk(0, rs[0].getvalue())
else:
# dest is UDX op, use jobid as the key
self.result[string.atoi(jobid)] = rs
if self.is_checkpoint == 1:
self.write_data_to_disk(0, rs[0].getvalue())
else:
# use partid as the key
for i in range(len(rs)):
if i not in self.result:
self.result[i] = [rs[i]]
else:
self.result[i].append(rs[i])
if self.is_checkpoint == 1:
for i in range(len(rs)):
self.write_data_to_disk(i, rs[i].getvalue())
# check if the whole data exceeds the LIMITATION
if rs_type != self.MULTI_FILE:
if self.is_checkpoint is not None and self.is_checkpoint == conf.CHECKPOINT or self.total_size > self.MAX_SIZE:
for dataid in self.result:
data = ""
for d in self.result[dataid]:
data += d.getvalue()
self.write_data_to_disk(dataid, data)
self.result_type = self.MULTI_FILE
e_time = time.time()
if self.total_time == 0:
self.total_time = (e_time - s_time)
self.send_status_to_master(jobid, conf.PENDING)
elif exp.lower().startswith("create"):
ParaLiteLog.info("proc_create: START")
ParaLiteLog.info("SQL: %s" % exp)
self.proc_create(exp, target_db)
ParaLiteLog.info("proc_create: START")
self.send_status_to_master(jobid, conf.ACK)
self.is_running = False
elif exp.lower().startswith("drop"):
ParaLiteLog.info("proc_drop: START")
self.proc_drop(exp, target_db)
self.send_status_to_master(jobid, conf.ACK)
self.is_running = False
ParaLiteLog.debug("*****JOB %s******:finish" % jobid)
self.ex_w_time += (time.time() - self.ex_s_time)
self.ex_s_time = 0
elif m[0] == conf.JOB_END:
ParaLiteLog.debug("MESSAGE: %s" % message)
# all jobs are finished
# create a dictionary to store the status of each part of data
data_status = {} # {data_id:[(pos_in_result, status)]}
for dataid in self.result:
if dataid not in data_status:
data_status[dataid] = []
for i in range(len(self.result[dataid])):
data_status[dataid].append((i, 1))
self.data_status = data_status
self.reader = self.get_next_reader()
self.send_rs_info_to_master(self.total_size, self.total_time)
# distribute data
if self.dest == conf.DATA_TO_ONE_CLIENT:
self.distribute_data()
self.send_status_to_master(" ".join(self.job_data), conf.ACK)
self.is_running = False
elif self.dest == conf.DATA_TO_DB:
self.distribute_data()
elif m[0] == conf.DLOAD_REPLY:
reply = sep.join(m[1:])
ParaLiteLog.info("receive the information from the master")
ParaLiteLog.debug(reply)
if len(self.data.getvalue()) != 0:
dload_client.dload_client().load_internal_buffer(
reply, self.dest_table, self.data, self.fashion,
self.hash_key, self.hash_key_pos, self.db_col_sep,
self.db_row_sep, self.db_col_sep, False, "0", self.log_dir)
# send END_TAG to the master
client_id = "0"
msg = sep.join([conf.REQ, conf.END_TAG, gethostname(), client_id])
so_master = socket(AF_INET, SOCK_STREAM)
so_master.connect((self.master_name, self.master_port))
so_master.send("%10s%s" % (len(msg), msg))
so_master.close()
ParaLiteLog.debug("sending to master: %s" % (conf.END_TAG))
ParaLiteLog.debug("----- dload client finish -------")
elif m[0] == conf.DATA_PERSIST:
ParaLiteLog.debug("MESSAGE: %s" % message)
# if the data is requried to be persisted or not
self.process_ck_info(m)
elif m[0] == conf.DATA_DISTRIBUTE:
ParaLiteLog.debug("MESSAGE: %s" % message)
# send a part of data to the next operator
# DATA_DISTRIBUTE:partition_num:destnode
part_id, destnode = m[1:]
data = self.get_data_by_part_id(self.result, string.atoi(part_id))
# DATA message includes: type:id+data
# the first 2 chars represents the opid
msg = sep.join([conf.DATA, "%2s%s" % (self.opid, data)])
if destnode == gethostname():
# use local socket
addr = self.p_node[destnode][1]
t = AF_UNIX
else:
addr = (destnode, self.p_node[destnode][0])
t = AF_INET
self.send_data_to_node(msg, t, addr)
ParaLiteLog.debug("send data susscufully %s %s --> %s" % (self.opid, gethostname(), destnode))
elif m[0] == conf.DATA_DISTRIBUTE_UDX:
ParaLiteLog.debug("MESSAGE: %s" % message)
# send data to udx client
# m[1:] = worker.id:jobid:(node:port | addr):size
if len(m) == 6:
w_id, jobid = m[1:3]
addr = (m[3], string.atoi(m[4]))
t = AF_INET
bk = string.atoi(m[5])
elif len(m) == 5:
w_id, jobid = m[1:3]
addr = m[3]
t = AF_UNIX
bk = string.atoi(m[4])
data = self.get_data_by_blocksize(jobid, bk)
if not data:
# if we don't send something here, udx will not send KAL
# again, and then they will not receive data again, the whole
# process will be blocked for ever
msg = sep.join([conf.DATA, "EMPTY"])
else:
msg = sep.join([conf.DATA, data])
self.send_data_to_node(msg, t, addr)
elif m[0] == conf.DATA_REPLICA:
ParaLiteLog.debug("MESSAGE: %s" % message)
# message --> DATA_REPLICA:DATANODE:DATAID:DATA
datanode, dataid = m[1:3]
f_name = self.get_file_name_by_data_id(gethostname(), dataid)
fr = open(f_name, "wa")
fr.write(m[4])
fr.close()
elif m[0] == conf.NODE_FAIL:
ParaLiteLog.debug("MESSAGE: %s" % message)
# message --> NODE_FAIL:FAILED_NODE:REPLICA_NODE
failed_node, replica_node = m[1:3]
self.failed_node.append(failed_node)
if replica_node == gethostname():
# load replica data for the failed node
self.recovery_data(self.replica_result, replica_node)
except Exception, e:
es("in sql_proc : %s" % traceback.format_exc())
ParaLiteLog.info(traceback.format_exc())
self.is_running = False
self.no_error = False
def distribute(self, data):
sep = conf.SEPERATOR
if self.dest == conf.DATA_TO_CLIENTS:
# This is final result, store them to a file
dest_file = conf.OUTPUT_DIR + self.dest
file_handler = io.FileIO(dest_file, 'wb', closefd=True)
bw = io.BufferedWriter(file_handler, buffer_size=65536)
for row in data:
bw.write(row)
bw.write('\n')
bw.close()
addr = (conf.MASTER_NAME, string.atoi(conf.MASTER_PORT))
sock = socket(AF_INET, SOCK_STREAM)
sock.connect(addr)
msg = '%s:%s:%s:%s' % (conf.ACK, conf.DATA_NODE, self.cqid, gethostname())
sock.send('%10s%s' % (len(msg), msg))
elif self.dest == conf.DATA_TO_ANO_OP:
# This is intermediate data, pipeline it
num_of_dest = len(self.p_node)
if num_of_dest == 1:
port = self.p_node.keys()[0]
node = self.p_node[port]
queue = cqueue.cqueue(node, port)
queue.connect()
# The first message is the output attrs of this sql
msg = ''
for out in self.output:
msg += '%s%s' % (out, sep)
queue.put(msg)
queue.put(data.getvalue())
queue.close()
elif num_of_dest > 1:
queue_list = []
"""
for p in self.p_port:
queue = cqueue(p.split(':')[0],string.atoi(p.split(':')[1]))
queue.connect()
# The first message is the output attrs of this sql
msg = ''
for out in self.output:
msg += '%s%s' % (out, sep)
queue.put(msg)
queue_list.append(queue)
"""
for port in self.p_node:
queue = cqueue(self.p_node[port],port)
queue.connect()
# The first message is the output attrs of this sql
msg = ''
for out in self.output:
msg += '%s%s' % (out, sep)
queue.put(msg)
queue_list.append(queue)
if self.split_key != None:
# partition data in hash fashion
split_key = self.split_key
if columns[0].find('.') == -1 and len(split_key.split('.')) == 2:
pos = columns.index(split_key.split('.')[1])
else:
pos = columns.index(split_key)
for row in rs:
partition_num = abs(hash(row[pos])) % num_of_dest
queue = queue_list[partition_num]
msg = ''
for r in data:
msg += '%s%s' % (str(r), sep)
queue.put(msg)
else:
# partitioning data in range fashion
for i in range(len(data)):
partition_num = i % num_of_dest
queue = queue_list[partition_num]
msg = ''
for r in data:
msg += '%s%s' % (str(r), sep)
queue.put(msg)
for queue in queue_list:
queue.close()
elif self.dest == conf.DATA_TO_ONE_CLIENT:
sock = socket(AF_INET, SOCK_STREAM)
sock.connect(self.client)
sock.send('%10sDATA' % (4))
sock.send("%10s%s" % (len(data.getvalue()), data.getvalue()))
sock.close()
self.notify_to_master()
elif self.dest == conf.DATA_TO_DB:
"""
out_name = "%s%sgroup_%s" % (LOG_DIR, os.sep, self.cqid)
f = open(out_name, "wb")
f.write(data.getvalue())
f.close()
"""
col_sep = self.db_col_sep
row_sep = self.db_row_sep
master = (self.master_name, self.master_port)
dload_client.dload_client().load_internal(master, self.cqid, gethostname(),self.dest_db, self.dest_table, data, 1, self.fashion, self.hash_key, self.hash_key_pos, col_sep, row_sep, col_sep, False, "0", LOG_DIR)
self.notify_to_master()
#os.remove(out_name)
else:
random_num = random.randint(0, len(self.client_sock) - 1)
addr = self.client_sock[random_num]
sock = socket(AF_INET, SOCK_STREAM)
sock.connect(addr)
d = string.strip(data.getvalue())
sock.send("%10s%s" % (len(d), d))
sock.close()
self.notify_to_master()
