def processFile(con,query,file,isType): i = 0 records=[] file.readline() # skip the first line. while True: line = file.readline() if not line: if len(records)>0: mydb.executeQueryRecords(con,query,records,False) records=[] break line=line.decode('raw_unicode_escape') ll = line.split(' ') record = None if isType: record=(ll[0],ll[2]) else: record = (ll[0],ll[1],ll[2]) records.append(record) i+=1 if i%10000 == 0: print '%d records inserted!' % i if len(records)>100: mydb.executeQueryRecords(con,query,records,False) records=[]
def processFile(con, query, file, isType): i = 0 records = [] file.readline() # skip the first line. while True: line = file.readline() if not line: if len(records) > 0: mydb.executeQueryRecords(con, query, records, False) records = [] break line = line.decode('raw_unicode_escape') ll = line.split(' ') record = None if isType: record = (ll[0], ll[2]) else: record = (ll[0], ll[1], ll[2]) records.append(record) i += 1 if i % 10000 == 0: print '%d records inserted!' % i if len(records) > 100: mydb.executeQueryRecords(con, query, records, False) records = []
def insertTranslation(con, table_name, fileName, lang): cn_lang = {'EN': 'en', 'RU': 'ru', 'FA': 'fa', 'ES': 'es'} prefix = '/c/' + cn_lang[lang] + '/' en_prefix = '/c/' + cn_lang['EN'] + '/' f = open(fileName, 'r') f.readline() records = [] buffer_size = 100 query = 'insert into __table_name__(rel,s,e) values(%s,%s,%s)' query = query.replace('__table_name__', table_name) i = 0 for line in f: line = line.decode('utf8') ll = line.split('\t') tri = ll[0][4:-1] tris = tri.split(',') # check for language if not (tris[0] == '/r/TranslationOf/' and (tris[1].startswith(prefix) and tris[2].startswith(en_prefix))): continue i += 1 if i % 10000 == 0: print i record = (tris[0], tris[1], tris[2]) records.append(record) if len(records) > buffer_size: mydb.executeQueryRecords(con, query, records, False) records = [] if len(records) > 0: mydb.executeQueryRecords(con, query, records, False) records = [] print i return i
native_property = '__total__' else: native_property = findLabel(originTree, property[1:-1], slang, 1) if native_property is None: native_property = findLabel(originTree, property[1:-1], slang, 2) if native_property is None: native_property = 'None' hit = properties[property] record = (typeName, property, native_type, native_property, hit, total) records.append(record) if len(records) > 100: mydb.executeQueryRecords(con, insertQuery, records, False) records = [] # break if len(records) > 0: mydb.executeQueryRecords(con, insertQuery, records, False) records = [] else: # lang == 'FA' typeFile = open('types_fa.txt', 'r') while True: line = typeFile.readline() if not line: break typeName = line.strip() properties = calculate.calculate(typeName, lang) total = properties['__total__'] for property in properties:
for property in properties: native_property = '' if property == '__total__': native_property = '__total__' else: native_property = findLabel(originTree,property[1:-1],slang,1) if native_property == None: native_property = findLabel(originTree,property[1:-1],slang,2) if native_property == None: native_property = 'None' hit = properties[property] record = (typeName,property,native_type,native_property,hit,total) records.append(record) if len(records)>100: mydb.executeQueryRecords(con,insertQuery,records,False) records=[] #break if len(records)>0: mydb.executeQueryRecords(con,insertQuery,records,False) records=[] else: # lang == 'FA' typeFile = open('types_fa.txt','r') while True: line = typeFile.readline() if not line: break typeName = line.strip() properties = calculate.calculate(typeName,lang) total = properties['__total__'] for property in properties: