def receive(connargs, shell_name, command):
bo_url = "http://" + connargs["host"] + ":" + str(connargs["port"]) + "/cmd"
workspace_name = connargs["workspace"]
cmd1 = command.split("\"")[0].split(" ")[0]
cmd2 = command.split("\"")[1]
#print "cmd1 :" , cmd1
#print "cmd2 :" , cmd2
proc = Popen(cmd2, shell=True ,stdout=PIPE)
insert_stmt = ""
for line in iter(proc.stdout.readline,''):
#print "::::" , line.rstrip()
#print line.strip()
if insert_stmt != "":
insert_stmt = insert_stmt + ","
if len(line.split()) == 0:
insert_stmt = insert_stmt + "(\"" + line + "\")"
else:
insert_stmt = insert_stmt + "(\"" + "\",\"".join([str(col) for col in line.split(',')]) + "\")"
#print ",".join([str(wtf) for wtf in p])
insert_stmt = "INSERT INTO " + cmd1 + " VALUES" + insert_stmt
print rpcshell.shell(connargs, "" , insert_stmt)
def send_return(connargs, shell_name, command):
bo_url = "http://" + connargs["host"] + ":" + str(connargs["port"]) + "/cmd"
workspace_name = connargs["workspace"]
cmd1 = command.split("\"")[1]
cmd2 = command.split("\"")[3]
ret_table = (command.split("\"")[4]).split(" ")[3]
#print cmd1
#print cmd2
#print ret_table
sneder = Popen(cmd2, shell=True, stdin=PIPE, stdout=PIPE)
res = return_Data(bo_url , cmd1 , True , workspace_name , connargs["timeout"])
#print res
out, err = sneder.communicate(input=res)
sneder.wait()
insert_stmt = ""
for line in out.split('\n'):
#print "::::" , line.rstrip()
#if line != "":
# print line
#else:
# break
if line == "":
break
if insert_stmt != "":
insert_stmt = insert_stmt + ","
if len(line.split()) == 0:
insert_stmt = insert_stmt + "(\"" + line + "\")"
else:
insert_stmt = insert_stmt + "(\"" + "\",\"".join([str(col) for col in line.split(',')]) + "\")"
#print ",".join([str(col) for col in line.split()])
#print ",".join([str(wtf) for wtf in p])
insert_stmt = "INSERT INTO " + ret_table + " VALUES" + insert_stmt
print rpcshell.shell(connargs, "" , insert_stmt)
def csvload(connargs, csv_file, bt_name ,insert_line=30000, noncheck_mod=False):
errorfile = open('errorinsert.txt', 'w')
addr_info = connargs['origin']
import time
now = time.time()
try:
f = open(csv_file,'rU')
except IOError:
print 'cannot open file'
return
except:
return
print ('open ' + csv_file + ' insert to ' + bt_name)
insert_prefix = 'INSERT INTO ' + bt_name + " VALUES "
line_count = 0
import csv
reader=csv.reader(f)
data_str = ""
animate={1:"|",2:"/",3:"-",0:"\\"}
for line in reader:
line_count += 1
buff=replaceSymbol(line)
try:
ascii_check(".".join(buff))
except:
buff=forcedecoding(buff)
if noncheck_mod == False:
data_str += "('"+"','".join(filterOutNonDisplayable(buff)) + "')"
else:
data_str += "('"+"','".join(buff) + "')"
if (line_count % insert_line) == 0 and line_count != 1:
try:
#print ("######" + json.dumps(data_str, ensure_ascii=False)[1:-1] )
rpcshell.shell(connargs, "" , insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1])
except:
traceback.print_exc()
print "Unexpected error:" + str(sys.exc_info()[0])
print line_count
errorfile.write(data_str)
errorfile.write('\n=========================================\n')
return
data_str = ""
else:
data_str += ","
sys.stdout.write('\r')
animate_token=animate[line_count%4]
sys.stdout.write(animate_token+" processing: "+str(line_count)+" lines")
sys.stdout.flush()
if (line_count % insert_line) != 0:
try:
#insert_stmt_all = insert_prefix + json.dumps(data_str[:len(data_str)-1], ensure_ascii=False)
insert_stmt_all = insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1]
#print (insert_stmt_all)
rpcshell.shell(connargs, "" , insert_stmt_all)
except:
traceback.print_exc()
print "Unexpected error:" + str(sys.exc_info()[0])
f.close()
end = time.time()
return '\ninsertion done. total %i rows , time: %ss' % (line_count, round((end - now),2))
