def ReadData(self):
if self.isDebug:
debugStartTime = time.time()
if not self.firstExecute: self.sock.SendMessage("CONT\r\n")
else: self.firstExecute = False
msg = self.sock.Readline()
try:
(tag, param) = msg.strip().split(" ", 1)
except:
tag = msg.strip()
if "+OK" == tag:
self.has_next = True
raise StopIteration
if "-" == tag[0]: raise Exception, param
data = self.sock.Read(int(param))
self.buffer = json.loads(data)
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] ReadData() %f" % (debugEndTime -
debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] ReadData() %f" %
(debugEndTime - debugStartTime))
def Login(self, id, password, libinfo):
if self.isDebug:
debugStartTime = time.time()
param = "%s,%s,%s" % (id, password, libinfo)
sendMsg = "LOGIN %s\r\n" % (base64.b64encode(param))
# send LOGIN command
self.sock.SendMessage(sendMsg)
# welcome message from PGD
msg = self.sock.Readline()
if msg[0] != "+": raise LoginFailException, msg
# welcome message from UDM
msg = self.sock.Readline()
if msg[0] != "+": raise ConnectionFailException, msg
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] Login() %f" % (debugEndTime -
debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] Login() %f" %
(debugEndTime - debugStartTime))
def Fetchall(self):
if self.isDebug:
debugStartTime = time.time()
if not self.firstExecute: self.sock.SendMessage("CONT ALL\r\n")
else: self.firstExecute = False
tmp_str = ""
tmp_list = []
while True:
msg = self.sock.Readline()
try:
(tag, param) = msg.strip().split(" ", 1)
except:
tag = msg.strip()
if "+OK" == tag:
break
if "-" == tag[0]:
raise Exception, param
tmp_list += json.loads(self.sock.Read(int(param)))
self.sock.SendMessage("CONT ALL\r\n")
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] Fetchall() %f" % (debugEndTime -
debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] Fetchall() %f" %
(debugEndTime - debugStartTime))
return tmp_list
def Connect(self):
if self.isDebug:
debugStartTime = time.time()
# ip
ip = self.addr_info[0]
# port
if len(self.addr_info) > 1:
port = self.addr_info[1]
else:
port = LISTENER_PORT
self.__connect(ip, port)
if self.isDirect:
(udm_ip, udm_port) = self.nsdConnect()
self.__connect(udm_ip, udm_port)
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == 'STDOUT':
print("[DEBUG_TIME] Connect() %f") % (debugEndTime -
debugStartTime)
elif self.LogModule == 'MOBIGEN':
__LOG__.Trace("[DEBUG_TIME] Connect() %f" %
(debugEndTime - debugStartTime))
def Main(argv):
logFile = "%s/MSD.log" % (Default.M6_LOG_DIR)
# FIXME : ADD MSD log option to Default
#Log.Init( Log.CRotatingLog (logFile, Default.MSD_LOG_MAX_LINE, Default.MSD_LOG_MAX_FILE))
Log.Init(
Log.CRotatingLog(logFile, Default.BIM_LOG_MAX_LINE,
Default.BIM_LOG_MAX_FILE))
server = TCPThreadServer.Server(9999, MSD.MSD, None)
#__LOG__.Trace("Start BSD (port=%s)" % Default.PORT["BMD"])
__LOG__.Trace("Start MSD (port=9999)")
server.start()
def Execute(self, sql):
if self.isDebug:
debugStartTime = time.time()
sql_length = len(sql)
sendMsg = "EXECUTE %s\r\n%s" % (sql_length, sql)
#print sendMsg;
self.firstExecute = False
self.sock.SendMessage(sendMsg)
self.firstExecute = True
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] Execute() %f" % (debugEndTime - debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] Execute() %f" % (debugEndTime - debugStartTime))
def Execute2(self, sql):
if self.isDebug:
debugStartTime = time.time()
sendMsg = "EXECUTE2 %s\r\n%s" % (len(sql), sql)
self.sock.SendMessage(sendMsg)
msg = self.sock.Readline()
if msg[0] != "+" : raise Exception, msg
self.firstExecute = False
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] Execute2() %f" % (debugEndTime - debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] Execute2() %f" % (debugEndTime - debugStartTime))
return msg
def SetRecordSep(self, sep) :
if self.isDebug:
debugStartTime = time.time()
sendMsg = "SET_RECORD_SEP %s\r\n" % (base64.b64encode(sep))
self.sock.SendMessage(sendMsg)
msg = self.sock.Readline()
if msg[0] != "+" : return False
self.record_sep = sep
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] SetRecordSep() %f" % (debugEndTime - debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] SetRecordSep() %f" % (debugEndTime - debugStartTime))
return True
def SetFieldSep(self, sep) :
if self.isDebug:
debugStartTime = time.time()
sendMsg = 'SET_FIELD_SEP %s\r\n' % base64.b64encode(sep.encode('utf-8')).decode('utf-8')
self.sock.SendMessage(sendMsg)
msg = self.sock.Readline()
if msg[0] != "+" : return False
self.field_sep = sep
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print("[DEBUG_TIME] SetFieldSep() %f" % (debugEndTime - debugStartTime))
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] SetFieldSep() %f" % (debugEndTime - debugStartTime))
return True
def SetInfo(self, id, password, host, libinfo) :
if self.isDebug:
debugStartTime = time.time()
param = "%s,%s,%s,%s" % (id, password, host, libinfo)
sendMsg = "SETINFO %s\r\n" % (base64.b64encode(param))
# send SETINFO command
self.sock.SendMessage(sendMsg)
# result message from UDM
msg = self.sock.Readline()
if msg[0] != "+" : raise ConnectionFailException, msg
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] SetInfo() %f" % (debugEndTime - debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] SetInfo() %f" % (debugEndTime - debugStartTime))
def Execute2(self, sql):
self.has_next = False
if self.isDebug:
debugStartTime = time.time()
# sql = self.check_semi(sql)
sendMsg = "EXECUTE2 %s\r\n%s" % (len(sql.encode('utf-8')), sql)
self.sock.SendMessage(sendMsg)
msg = self.sock.Readline()
if msg[0] != "+" : raise Exception(msg)
self.firstExecute = False
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print("[DEBUG_TIME] Execute2() %f" % (debugEndTime - debugStartTime))
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] Execute2() %f" % (debugEndTime - debugStartTime))
return msg
def Fetchall(self):
if self.isDebug:
debugStartTime = time.time()
if not self.firstExecute : self.sock.SendMessage("CONT ALL\r\n")
else: self.firstExecute = False
tmp_str = ""
while True:
msg = self.sock.Readline()
try: (tag, param) = msg.strip().split(" ", 1)
except: tag = msg.strip()
if "+OK" == tag:
break
if "-" == tag[0]:
raise Exception, param
if tmp_str == "":
tmp_str = self.sock.Read(int(param))
else:
tmp_str += self.record_sep + self.sock.Read(int(param))
self.sock.SendMessage("CONT ALL\r\n")
st = time.time()
tmp_list = tmp_str.split(self.record_sep)
tmp_list = map(lambda x: x.split(self.field_sep), tmp_list)
et = time.time()
#print "string to list : %f" % (et-st)
if self.isDebug:
debugEndTime = time.time()
if self.LogModule == "STDOUT":
print "[DEBUG_TIME] Fetchall() %f" % (debugEndTime - debugStartTime)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG_TIME] Fetchall() %f" % (debugEndTime - debugStartTime))
return tmp_list
def LoadCore(self, table, key, partition, control_file_path, data_file, CMD, data="", load_ready_check=False, reload=False) :
if load_ready_check:
retMsg = self.LoadReady(table, key, partition)
if retMsg[0] == '-': return retMsg
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeStart (%s)" % control_file_path,
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeStart (%s)" % control_file_path)
ctl_size = os.path.getsize(control_file_path)
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeEnd"
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeEnd")
if data_file == "":
data_size = len(data)
else:
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeStart (%s)" % data_file,
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeStart (%s)" % data_file)
data_size = os.path.getsize(data_file)
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeEnd"
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeEnd")
sendMsg = None
if CMD == 'IMPORT':
sendMsg = "%s %s,%s,%s,%s,%s,%s\r\n" % (CMD, table, key, partition, ctl_size, data_size, str(reload).upper())
else:
sendMsg = "%s %s,%s,%s,%s,%s\r\n" % (CMD, table, key, partition, ctl_size, data_size)
self.sock.SendMessage(sendMsg)
if data_file == "":
fobj = open(control_file_path)
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] OpenControlFile %s" % control_file_path
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] OpenControlFile %s" % control_file_path)
while True:
if self.isDebug:
self.statusInfo = "Befor FileRead"
buf = fobj.read(self.bufSize)
if self.isDebug:
self.statusInfo = "Afefor FileRead"
if not buf: break #EOF
if self.isDebug:
self.statusInfo = "Befor SendMessage"
self.sock.SendMessage(buf)
if self.isDebug:
self.statusInfo = "After SendMessage"
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] End (%s)" % control_file_path
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] End (%s)" % control_file_path )
fobj.close()
self.sock.SendMessage(data)
else:
fileList = [control_file_path, data_file]
for f in fileList :
fobj = open(f)
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] OpenFile (%s)" % f
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] OpenFile (%s)" % f )
while True :
if self.isDebug:
self.statusInfo = "Befor FileRead"
buf = fobj.read(self.bufSize)
if self.isDebug:
self.statusInfo = "After FileRead"
if not buf: break #EOF
if self.isDebug:
self.statusInfo = "Befor SendMessage"
self.sock.SendMessage(buf)
if self.isDebug:
self.statusInfo = "After SendMessage"
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] End (%s)" % f
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] End (%s)" % f )
fobj.close()
def LoadCore(self,
table,
key,
partition,
control_file_path,
data_file,
CMD,
control_data=None,
data="",
load_ready_check=False,
reload=False,
csv_mode=False):
if load_ready_check:
retMsg = self.LoadReady(table, key, partition)
if retMsg[0] == '-': return retMsg
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeStart (%s)" % control_file_path,
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeStart (%s)" % control_file_path)
# check record_sep is valid when csv loading.
if csv_mode and not self.record_sep in ('\r', '\n'):
raise LoadTypeException, "-ERR Record Separator is invalid. [%s]\r\n" % repr(
self.record_sep)
# check control file is exists when normal loading.
if not csv_mode and (not (control_file_path or control_data)):
raise LoadTypeException, "-ERR ctl file doesn't exist.\r\n"
if control_file_path:
ctl_size = os.path.getsize(control_file_path)
elif control_data:
ctl_size = len(control_data)
else:
ctl_size = 4 # send 'NULL' if control_file_path is empty.
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeEnd"
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeEnd")
if data_file == "":
data_size = len(data)
else:
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeStart (%s)" % data_file,
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeStart (%s)" % data_file)
data_size = os.path.getsize(data_file)
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] GetSizeEnd"
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] GetSizeEnd")
sendMsg = None
if CMD == 'IMPORT':
if csv_mode:
sendMsg = "%s %s,%s,%s,%s,%s,%s,%s\r\n" % (
CMD, table, key, partition, ctl_size, data_size,
str(reload).upper(), str(csv_mode).upper())
else:
sendMsg = "%s %s,%s,%s,%s,%s,%s\r\n" % (
CMD, table, key, partition, ctl_size, data_size,
str(reload).upper())
else:
sendMsg = "%s %s,%s,%s,%s,%s\r\n" % (CMD, table, key, partition,
ctl_size, data_size)
self.sock.SendMessage(sendMsg)
fileList = []
if control_file_path:
fileList.append(control_file_path)
elif control_data:
self.sock.SendMessage(control_data)
else:
# send 'NULL' if control_file_path is empty.
self.sock.SendMessage('NULL')
if data_file:
fileList.append(data_file)
for f in fileList:
fobj = open(f)
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] OpenFile (%s)" % f
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] OpenFile (%s)" % f)
while True:
if self.isDebug:
self.statusInfo = "Befor FileRead"
buf = fobj.read(self.bufSize)
if self.isDebug:
self.statusInfo = "After FileRead"
if not buf: break #EOF
if self.isDebug:
self.statusInfo = "Befor SendMessage"
self.sock.SendMessage(buf)
if self.isDebug:
self.statusInfo = "After SendMessage"
if self.isDebug:
if self.LogModule == "STDOUT":
print "[DEBUG] End (%s)" % f
elif self.LogModule == "MOBIGEN":
__LOG__.Trace("[DEBUG] End (%s)" % f)
fobj.close()
if not data_file:
self.sock.SendMessage(data)
def _logger(self, msg):
if self.isDebug:
if self.LogModule == "STDOUT":
print(msg)
elif self.LogModule == "MOBIGEN":
__LOG__.Trace(msg)
