def printLine(csv_file): try: f=open(csv_file,'rU') except: return import csv reader=csv.reader(f) linecount=0; for line in reader: linecount+=1 buff=replaceSymbol(line,linecount=linecount) try: ascii_check(",".join(buff)) except: print "error:", ",".join(forcedecoding(buff,linecount=linecount)) #print str(linecount), buff #print buff f.close()
def csvload(connargs, csv_file, bt_name ,insert_line=30000, noncheck_mod=False): errorfile = open('errorinsert.txt', 'w') addr_info = connargs['origin'] import time now = time.time() try: f = open(csv_file,'rU') except IOError: print 'cannot open file' return except: return print ('open ' + csv_file + ' insert to ' + bt_name) insert_prefix = 'INSERT INTO ' + bt_name + " VALUES " line_count = 0 import csv reader=csv.reader(f) data_str = "" animate={1:"|",2:"/",3:"-",0:"\\"} for line in reader: line_count += 1 buff=replaceSymbol(line) try: ascii_check(".".join(buff)) except: buff=forcedecoding(buff) if noncheck_mod == False: data_str += "('"+"','".join(filterOutNonDisplayable(buff)) + "')" else: data_str += "('"+"','".join(buff) + "')" if (line_count % insert_line) == 0 and line_count != 1: try: #print ("######" + json.dumps(data_str, ensure_ascii=False)[1:-1] ) rpcshell.shell(connargs, "" , insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1]) except: traceback.print_exc() print "Unexpected error:" + str(sys.exc_info()[0]) print line_count errorfile.write(data_str) errorfile.write('\n=========================================\n') return data_str = "" else: data_str += "," sys.stdout.write('\r') animate_token=animate[line_count%4] sys.stdout.write(animate_token+" processing: "+str(line_count)+" lines") sys.stdout.flush() if (line_count % insert_line) != 0: try: #insert_stmt_all = insert_prefix + json.dumps(data_str[:len(data_str)-1], ensure_ascii=False) insert_stmt_all = insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1] #print (insert_stmt_all) rpcshell.shell(connargs, "" , insert_stmt_all) except: traceback.print_exc() print "Unexpected error:" + str(sys.exc_info()[0]) f.close() end = time.time() return '\ninsertion done. total %i rows , time: %ss' % (line_count, round((end - now),2))