def printLine(csv_file):
try:
f=open(csv_file,'rU')
except:
return
import csv
reader=csv.reader(f)
linecount=0;
for line in reader:
linecount+=1
buff=replaceSymbol(line,linecount=linecount)
try:
ascii_check(",".join(buff))
except:
print "error:", ",".join(forcedecoding(buff,linecount=linecount))
#print str(linecount), buff
#print buff
f.close()
def csvload(connargs, csv_file, bt_name ,insert_line=30000, noncheck_mod=False):
errorfile = open('errorinsert.txt', 'w')
addr_info = connargs['origin']
import time
now = time.time()
try:
f = open(csv_file,'rU')
except IOError:
print 'cannot open file'
return
except:
return
print ('open ' + csv_file + ' insert to ' + bt_name)
insert_prefix = 'INSERT INTO ' + bt_name + " VALUES "
line_count = 0
import csv
reader=csv.reader(f)
data_str = ""
animate={1:"|",2:"/",3:"-",0:"\\"}
for line in reader:
line_count += 1
buff=replaceSymbol(line)
try:
ascii_check(".".join(buff))
except:
buff=forcedecoding(buff)
if noncheck_mod == False:
data_str += "('"+"','".join(filterOutNonDisplayable(buff)) + "')"
else:
data_str += "('"+"','".join(buff) + "')"
if (line_count % insert_line) == 0 and line_count != 1:
try:
#print ("######" + json.dumps(data_str, ensure_ascii=False)[1:-1] )
rpcshell.shell(connargs, "" , insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1])
except:
traceback.print_exc()
print "Unexpected error:" + str(sys.exc_info()[0])
print line_count
errorfile.write(data_str)
errorfile.write('\n=========================================\n')
return
data_str = ""
else:
data_str += ","
sys.stdout.write('\r')
animate_token=animate[line_count%4]
sys.stdout.write(animate_token+" processing: "+str(line_count)+" lines")
sys.stdout.flush()
if (line_count % insert_line) != 0:
try:
#insert_stmt_all = insert_prefix + json.dumps(data_str[:len(data_str)-1], ensure_ascii=False)
insert_stmt_all = insert_prefix + json.dumps(data_str, ensure_ascii=False)[1:-1]
#print (insert_stmt_all)
rpcshell.shell(connargs, "" , insert_stmt_all)
except:
traceback.print_exc()
print "Unexpected error:" + str(sys.exc_info()[0])
f.close()
end = time.time()
return '\ninsertion done. total %i rows , time: %ss' % (line_count, round((end - now),2))
