def receive(connargs, shell_name, command): bo_url = "http://" + connargs["host"] + ":" + str(connargs["port"]) + "/cmd" workspace_name = connargs["workspace"] cmd1 = command.split("\"")[0].split(" ")[0] cmd2 = command.split("\"")[1] #print "cmd1 :" , cmd1 #print "cmd2 :" , cmd2 proc = Popen(cmd2, shell=True ,stdout=PIPE) insert_stmt = "" for line in iter(proc.stdout.readline,''): #print "::::" , line.rstrip() #print line.strip() if insert_stmt != "": insert_stmt = insert_stmt + "," if len(line.split()) == 0: insert_stmt = insert_stmt + "(\"" + line + "\")" else: insert_stmt = insert_stmt + "(\"" + "\",\"".join([str(col) for col in line.split(',')]) + "\")" #print ",".join([str(wtf) for wtf in p]) insert_stmt = "INSERT INTO " + cmd1 + " VALUES" + insert_stmt print rpcshell.shell(connargs, "" , insert_stmt)
def send_return(connargs, shell_name, command): bo_url = "http://" + connargs["host"] + ":" + str(connargs["port"]) + "/cmd" workspace_name = connargs["workspace"] cmd1 = command.split("\"")[1] cmd2 = command.split("\"")[3] ret_table = (command.split("\"")[4]).split(" ")[3] #print cmd1 #print cmd2 #print ret_table sneder = Popen(cmd2, shell=True, stdin=PIPE, stdout=PIPE) res = return_Data(bo_url , cmd1 , True , workspace_name , connargs["timeout"]) #print res out, err = sneder.communicate(input=res) sneder.wait() insert_stmt = "" for line in out.split('\n'): #print "::::" , line.rstrip() #if line != "": # print line #else: # break if line == "": break if insert_stmt != "": insert_stmt = insert_stmt + "," if len(line.split()) == 0: insert_stmt = insert_stmt + "(\"" + line + "\")" else: insert_stmt = insert_stmt + "(\"" + "\",\"".join([str(col) for col in line.split(',')]) + "\")" #print ",".join([str(col) for col in line.split()]) #print ",".join([str(wtf) for wtf in p]) insert_stmt = "INSERT INTO " + ret_table + " VALUES" + insert_stmt print rpcshell.shell(connargs, "" , insert_stmt)
def csvload(connargs, csv_file, bt_name ,insert_line=30000, noncheck_mod=False): errorfile = open('errorinsert.txt', 'w') addr_info = connargs['origin'] import time now = time.time() try: f = open(csv_file,'rU') except IOError: print 'cannot open file' return except: return print ('open ' + csv_file + ' insert to ' + bt_name) insert_prefix = 'INSERT INTO ' + bt_name + " VALUES " line_count = 0 import csv reader=csv.reader(f) data_str = "" animate={1:"|",2:"/",3:"-",0:"\\"} for line in reader: line_count += 1 buff=replaceSymbol(line) try: ascii_check(".".join(buff)) except: buff=forcedecoding(buff) if noncheck_mod == False: data_str += "('"+"','".join(filterOutNonDisplayable(buff)) + "')" else: data_str += "('"+"','".join(buff) + "')" if (line_count % insert_line) == 0 and line_count != 1: try: #print ("######" + json.dumps(data_str, ensure_ascii=False)[1:-1] ) rpcshell.shell(connargs, "" , insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1]) except: traceback.print_exc() print "Unexpected error:" + str(sys.exc_info()[0]) print line_count errorfile.write(data_str) errorfile.write('\n=========================================\n') return data_str = "" else: data_str += "," sys.stdout.write('\r') animate_token=animate[line_count%4] sys.stdout.write(animate_token+" processing: "+str(line_count)+" lines") sys.stdout.flush() if (line_count % insert_line) != 0: try: #insert_stmt_all = insert_prefix + json.dumps(data_str[:len(data_str)-1], ensure_ascii=False) insert_stmt_all = insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1] #print (insert_stmt_all) rpcshell.shell(connargs, "" , insert_stmt_all) except: traceback.print_exc() print "Unexpected error:" + str(sys.exc_info()[0]) f.close() end = time.time() return '\ninsertion done. total %i rows , time: %ss' % (line_count, round((end - now),2))