def insertTokenFunc(con,pairs,query): records = [] for pair in pairs: reasons = [] consequences = [] id = '' for sent in pair: if id == '': id = sent.id.split('_')[0] sid = sent.id.split('_')[1] id += '_'+sid if sent.rc == 'R': reasons.append(sent) elif sent.rc == 'C': consequences.append(sent) jsonDict = {} jsonDict['id']=id jsonDict['r']=[sent.tokens for sent in reasons] jsonDict['c']=[sent.tokens for sent in consequences] jsonStr = json.dumps(jsonDict) record = (id,jsonStr) records.append(record) if len(records) >100: mydb.executeQueryRecords(con,query,records,False) records = [] if len(records) >0: mydb.executeQueryRecords(con,query,records,False) records = []
def insert(con,fileName,query,name,reverse): file = open(fileName,'r') records=[] for line in file: line = line.strip() r = json.loads(line) id = int(r['id']) pair = json.dumps(r[name]) record = (id,pair) if reverse: record = (pair,id) records.append(record) if len(records) >100: mydb.executeQueryRecords(con,query,records,False) records = [] if len(records) >0: mydb.executeQueryRecords(con,query,records,False) records = []
def insertCoref(fileName): con = mydb.getCon(CONN_STRING) json_date = open(fileName) data = json.load(json_date) i = 0 records = [] query = "insert into coref(id, replace) values(%s,%s);" for key in data: if i % 1000 == 0: print i i += 1 id = key txt = json.dumps(data[key]) records.append((id,txt)) if len(records) >= 100: try: mydb.executeQueryRecords(con, query,records, False) except psycopg2.DatabaseError, e: print 'Error %s' % e records = []
if i % 1000 == 0: print i i += 1 id = key txt = json.dumps(data[key]) records.append((id,txt)) if len(records) >= 100: try: mydb.executeQueryRecords(con, query,records, False) except psycopg2.DatabaseError, e: print 'Error %s' % e records = [] try: if len(records) >0: mydb.executeQueryRecords(con, query,records,False) except psycopg2.DatabaseError, e: print 'Error %s' % e json_date.close() con.close() # to clear coref table def clearTables(CONN_STRING): con = mydb.getCon(CONN_STRING) queries = list() queries.append('delete from coref') mydb.executeManyQuery(con, queries, False) con.close()