def insertTokenFunc(con,pairs,query):
records = []
for pair in pairs:
reasons = []
consequences = []
id = ''
for sent in pair:
if id == '':
id = sent.id.split('_')[0]
sid = sent.id.split('_')[1]
id += '_'+sid
if sent.rc == 'R':
reasons.append(sent)
elif sent.rc == 'C':
consequences.append(sent)
jsonDict = {}
jsonDict['id']=id
jsonDict['r']=[sent.tokens for sent in reasons]
jsonDict['c']=[sent.tokens for sent in consequences]
jsonStr = json.dumps(jsonDict)
record = (id,jsonStr)
records.append(record)
if len(records) >100:
mydb.executeQueryRecords(con,query,records,False)
records = []
if len(records) >0:
mydb.executeQueryRecords(con,query,records,False)
records = []
def insert(con,fileName,query,name,reverse):
file = open(fileName,'r')
records=[]
for line in file:
line = line.strip()
r = json.loads(line)
id = int(r['id'])
pair = json.dumps(r[name])
record = (id,pair)
if reverse:
record = (pair,id)
records.append(record)
if len(records) >100:
mydb.executeQueryRecords(con,query,records,False)
records = []
if len(records) >0:
mydb.executeQueryRecords(con,query,records,False)
records = []
def insertCoref(fileName):
con = mydb.getCon(CONN_STRING)
json_date = open(fileName)
data = json.load(json_date)
i = 0
records = []
query = "insert into coref(id, replace) values(%s,%s);"
for key in data:
if i % 1000 == 0:
print i
i += 1
id = key
txt = json.dumps(data[key])
records.append((id,txt))
if len(records) >= 100:
try:
mydb.executeQueryRecords(con, query,records, False)
except psycopg2.DatabaseError, e:
print 'Error %s' % e
records = []
if i % 1000 == 0:
print i
i += 1
id = key
txt = json.dumps(data[key])
records.append((id,txt))
if len(records) >= 100:
try:
mydb.executeQueryRecords(con, query,records, False)
except psycopg2.DatabaseError, e:
print 'Error %s' % e
records = []
try:
if len(records) >0:
mydb.executeQueryRecords(con, query,records,False)
except psycopg2.DatabaseError, e:
print 'Error %s' % e
json_date.close()
con.close()
# to clear coref table
def clearTables(CONN_STRING):
con = mydb.getCon(CONN_STRING)
queries = list()
queries.append('delete from coref')
mydb.executeManyQuery(con, queries, False)
con.close()
