Exemple #1
0
def main():
    offset = 0
    # change your output dir
    dir = 'evaluate'
    pfile = open(dir + '/evaluate.pairs.' + str(offset) + '.txt','w')
    tfile = open(dir + '/evaluate.tuples.'+ str(offset) + '.txt','w')
    cfile = open(dir + '/evaluate.corefs.'+ str(offset) + '.txt','w')
    
    CONN_STRING = mydb.get_CONN()
    con = mydb.getCon(CONN_STRING)
    
    # change the amount of results, currently is 200
    query = 'select * from rc order by id limit 200 offset 0'
    rows = mydb.executeQueryResult(con,query,False)
    
    for row in rows:
        id = int(row[0])
        pairs = json.loads(row[1])
        tuples = json.loads(row[2])
        coref = None
        if row[3]:
            coref = json.loads(row[3])
        
        # write the pairs
        for pair in pairs:
            reasons = pair[0]
            consequences = pair[1]
            for reason in reasons:
                pfile.write(str(id)+'\t'+reason[0]+'\n')
            for consequence in consequences:
                pfile.write(str(id)+'\t'+consequence[0]+'\n')
        
        # write the tuples:
        for t in tuples:
            reasons = t[0]
            consequences = t[1]
            for reason in reasons:
                tfile.write(str(id)+'\t'+tuple2str(reason[0])+'\n')
            for consequence in consequences:
                tfile.write(str(id)+'\t'+tuple2str(consequence[0])+'\n')

        # write the tuples 
        if row[3]:
            corefPairs=rcreplace.mergeCoref(pairs,coref)
            for pair in corefPairs:
                reasons = pair[0]
                consequences = pair[1]
            for reason in reasons:
                cfile.write(str(id)+'\t'+reason+'\n')
            for consequence in consequences:
                cfile.write(str(id)+'\t'+consequence+'\n')

    pfile.close()
    tfile.close()
    cfile.close()
Exemple #2
0
def calculate(type_name,lang):
    result = {}
    indexs={"EN":0,"ES":1,"RU":2,"FA":3}
    type_table_names=["type_en","type_es","type_ru"]
    property_table_names=["property_en","property_es","property_ru"]
    CONN_STRING = mydb.get_CONN()
    con = mydb.getCon(CONN_STRING)
    rows=[]
    property_table_name=''
    type_table_name = ''
    if lang !='FA':
        index = indexs[lang]
        type_table_name = type_table_names[index]
        property_table_name = property_table_names[index]
        type_name = '<'+type_name+'>'
        query = "select entity from __type_table__ where type = '__type_name__'"
        query = query.replace('__type_table__',type_table_name)
        query = query.replace('__type_name__',type_name)
        rows = mydb.executeQueryResult(con,query,True)
        result['__total__']=len(rows)
    else:
        property_table_name = 'property_fa'
        
        query = "select distinct entity from property_fa where property = '<http://fa.dbpedia.org/property/type>' and value ='__type_name__' "
        query = query.replace('__type_name__',type_name)
        rows = mydb.executeQueryResult(con,query,True)
        result['__total__']=len(rows)

    temp_result={}
    print len(rows)
    i=0
    for row in rows:
        entity = row[0]
        entity = entity.replace("'","''")
        query = "select property from __property_table__ where entity = '__entity_name__';"
        query = query.replace('__property_table__',property_table_name)
        query = query.replace('__entity_name__',entity)
        ps = mydb.executeQueryResult(con,query,False)
        for p in ps:
            if not p[0] in temp_result:
                temp_result[p[0]]=1
        for p in temp_result:
            if not p in result:
                result[p]=0
            result[p]+=1
        temp_result={}
        i+=1
        if i % 50 ==0:
            print i
    #print result
    return result
Exemple #3
0
def main():
    # file to save
    #jfile = open('result.tuple.json.txt','w')
    file_path = os.path.join(settings.PROJECT_DIR,'result/raw/result.sentence.json.txt')
    jsfile = open(file_path,'w')
    #file = open('result.txt','w')
    #jhfile = open('result.jh.txt','w')
    # db
    CONN_STRING = mydb.get_CONN()
    con = mydb.getCon(CONN_STRING)
    query = 'select id , review_clauses from review where review_clauses is not null order by id'
    rows = mydb.executeQueryResult(con,query,True)
    
    for row in rows:
        id = row[0]
        review = row[1]
        if not review:
            continue
        review = review.decode('utf-8')
        clauses = review.split('###')
        tpairs = processReview(clauses)
        if len(tpairs) == 0:
            continue
        #jfile.write(json.dumps({'id':id,'pairs':pairs})+'\n')
        jsfile.write(json.dumps({'id':id,'sen_pairs':tpairs})+'\n')
        
        #file.write('id:'+str(id)+'\n')
        #jhfile.write('id:'+str(id)+'\n')
        # for tpair in tpairs:
        #     file.write('Reasons:'+repr(tpair[0])+'\n')
        #     file.write('Consequences:'+ repr(tpair[1])+'\n')
            
        # for pair in pairs:
        #     jhfile.write('Reasons:\n')
        #     jhfile.write(repr(pair[0][0][0]['subj'])+' ')
        #     jhfile.write(repr(pair[0][0][0]['verb'])+' ')
        #     jhfile.write(repr(pair[0][0][0]['dobj'])+' ')
        #     jhfile.write(repr(pair[0][0][0]['iobj'])+'\n')
            
        #     jhfile.write('Consequences:\n')
        #     jhfile.write(repr(pair[1][0][0]['subj'])+' ')
        #     jhfile.write(repr(pair[1][0][0]['verb'])+' ')
        #     jhfile.write(repr(pair[1][0][0]['dobj'])+' ')
        #     jhfile.write(repr(pair[1][0][0]['iobj'])+'\n')
    jsfile.close()        
def main():

    from optparse import OptionParser
   
    # option
    usage="usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-t","--tfile", dest='type_filePath',help="Type file path")
    parser.add_option("-p","--pfile", dest='property_filePath',help="Property file path")
    parser.add_option("-l","--lang", dest='lang',help="language:EN,ES,RU,FA")
    parser.add_option("-d","--debug",dest = "debug" , action = "store_true",
                  help="output debug info, default is false", default = False)
   
    (options,args) = parser.parse_args()
    
    lang = options.lang
    tfilePath = options.type_filePath
    pfilePath = options.property_filePath
    debug = options.debug
    
    indexs={"EN":0,"ES":1,"RU":2,"FA":3}
    type_table_names=["type_en","type_es","type_ru","type_fa"]
    property_table_names=["property_en","property_es","property_ru","property_fa"]
    
    index=indexs[lang]
    type_table_name = type_table_names[index]
    property_table_name = property_table_names[index]

    CONN_STRING=mydb.get_CONN()
    con = mydb.getCon(CONN_STRING)
    # create table
    
    querys=[]
    querys.append( 'DROP TABLE IF EXISTS '+type_table_name+';' )
    querys.append('DROP TABLE IF EXISTS '+property_table_name+';')
    querys.append('CREATE TABLE '+type_table_name+'(entity varchar,type varchar);')
    querys.append('CREATE TABLE '+property_table_name+'(entity varchar,property varchar,value varchar);')
    mydb.executeManyQuery(con,querys,debug)
      
    # process the data
    tfile = open(tfilePath,'r')
    pfile = open(pfilePath,'r')
    # for the type file
    query_t='insert into __table_name__(entity,type) VALUES(%s, %s)'
    query_t=query_t.replace('__table_name__',type_table_name)
    processFile(con,query_t,tfile,True)
    # for property table
    query_p='insert into __table_name__(entity,property,value) VALUES(%s, %s, %s)'
    query_p=query_p.replace('__table_name__',property_table_name)
    processFile(con,query_p,pfile,False)
    
    # build index
    querys=[]
    query = 'create index __indexName__ on __index__;'
    querys.append(query.replace('__indexName__',type_table_name+'_entity').replace('__index__',type_table_name+'(entity)'))
    querys.append(query.replace('__indexName__',type_table_name+'_type').replace('__index__',type_table_name+'(type)'))
    querys.append(query.replace('__indexName__',property_table_name+'_entity').replace('__index__',property_table_name+'(entity)'))
    querys.append(query.replace('__indexName__',property_table_name+'_property').replace('__index__',property_table_name+'(property)'))
    mydb.executeManyQuery(con,querys,debug)
      
    # close db
    mydb.closeCon(con)
Exemple #5
0
def main():
    from optparse import OptionParser

    # option
    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-t",
                      "--tfile",
                      dest='type_filePath',
                      help="Type file path")
    parser.add_option("-p",
                      "--pfile",
                      dest='property_filePath',
                      help="Property file path")
    parser.add_option("-l",
                      "--lang",
                      dest='lang',
                      help="language: EN, ES, RU, FA")
    parser.add_option("-d",
                      "--debug",
                      dest="debug",
                      action="store_true",
                      help="output debug info, default is false",
                      default=False)

    (options, args) = parser.parse_args()

    lang = options.lang
    tfilePath = options.type_filePath
    pfilePath = options.property_filePath
    debug = options.debug

    indexes = {"EN": 0, "ES": 1, "RU": 2, "FA": 3}
    type_table_names = ["type_en", "type_es", "type_ru", "type_fa"]
    property_table_names = [
        "property_en", "property_es", "property_ru", "property_fa"
    ]

    index = indexes[lang]
    type_table_name = type_table_names[index]
    property_table_name = property_table_names[index]

    CONN_STRING = mydb.get_CONN()
    con = mydb.getCon(CONN_STRING)
    # create table

    queries = []
    queries.append('DROP TABLE IF EXISTS ' + type_table_name + ';')
    queries.append('DROP TABLE IF EXISTS ' + property_table_name + ';')
    queries.append('CREATE TABLE ' + type_table_name +
                   '(entity varchar,type varchar);')
    queries.append('CREATE TABLE ' + property_table_name +
                   '(entity varchar,property varchar,value varchar);')
    mydb.executeManyQuery(con, queries, debug)

    # process the data
    tfile = open(tfilePath, 'r')
    pfile = open(pfilePath, 'r')
    # for the type file
    query_t = 'insert into __table_name__(entity,type) VALUES(%s, %s)'
    query_t = query_t.replace('__table_name__', type_table_name)
    processFile(con, query_t, tfile, True)
    # for property table
    query_p = 'insert into __table_name__(entity,property,value) VALUES(%s, %s, %s)'
    query_p = query_p.replace('__table_name__', property_table_name)
    processFile(con, query_p, pfile, False)

    # build index
    queries = []
    query = 'create index __indexName__ on __index__;'
    queries.append(
        query.replace('__indexName__', type_table_name + '_entity').replace(
            '__index__', type_table_name + '(entity)'))
    queries.append(
        query.replace('__indexName__', type_table_name + '_type').replace(
            '__index__', type_table_name + '(type)'))
    queries.append(
        query.replace('__indexName__',
                      property_table_name + '_entity').replace(
                          '__index__', property_table_name + '(entity)'))
    queries.append(
        query.replace('__indexName__',
                      property_table_name + '_property').replace(
                          '__index__', property_table_name + '(property)'))
    mydb.executeManyQuery(con, queries, debug)

    # close db
    mydb.closeCon(con)
Exemple #6
0
#!/usr/bin/python
# -*- coding: utf-8 -*-

import mydb

CONN_STRING = mydb.get_CONN()
con = mydb.getCon(CONN_STRING)

query = "select distinct value from property_fa where property = '<http://fa.dbpedia.org/property/type>'"

rows = mydb.executeQueryResult(con, query, True)

file = open('types_fa.txt', 'w')

print len(rows)
for row in rows:
    file.write(row[0] + '\n')

file.close()
import mydb
import sys

typeFile = open('es_property.txt', 'r')
transFile = open('es_property_translate.txt', 'r')

CONN_STRING = mydb.get_CONN()
con = mydb.getCon(CONN_STRING)


while True:
    type = typeFile.readline()
    if not type:
        break
    trans = transFile.readline()
    type = type.strip()
    trans = trans.strip()
    query = "update statistic_es set native_property = '__trans__' where property = '__type__' "
    query = query.replace('__trans__', trans)
    query = query.replace('__type__', type)
    mydb.executeQuery(con, query, True)