def walk(self): ''' Walkthrough all persons in db. ''' while True: try: lastId = -1 # while True: conn = DB.pool().getConnection() cursor = conn.cursor() #print self.sql % (self.gen, start_id, self.limit) cursor.execute(self.sql, (self.gen, lastId, self.fetch_size)) data = cursor.fetchall() print "&-walker-:> walk person(citation) %s items" % cursor.rowcount if cursor.rowcount == 0: break id = 0 for id, fullname, pubcount in data: lastId = id person = Person(id, fullname, pubcount) self.processer(person) cursor.close() conn.close() # sleep 10 minutes next loop time.sleep(20) except MySQLdb.Error, e: #@UndefinedVariable ExceptionHelper.print_exec(e) # raise return data
def getPublicationByConf(self, jconf_id): '''Get all publications of a person''' while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( '''select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.startpage,\ p.endpage, p.ncitation, p.u_citation_gen from publication p where p.jconf=%s''', jconf_id) data = cursor.fetchall() pubs = [] for id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation, gen in data: pub = Publication(id, year, title, pubkey, jconf, authors, ncitation) pub.startpage = startpage pub.endpage = endpage pubs.append(pub) cursor.close() conn.close() return pubs except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) # sys.exit(1) return data
def walk(self): ''' Walkthrough all pubs in db. ''' while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() if(self.replace_sql): if(self.replace_sql_params): cursor.execute(self.replace_sql, self.replace_sql_params) else: cursor.execute(self.replace_sql) else: # id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation cursor.execute("select id, year, title, pubkey, jconf, authors, ncitation from publication where %s", self.sql_condition) data = cursor.fetchall() print "walk through %s items" % cursor.rowcount for id, year, title, pubkey, jconf, authors, ncitation in data: pub = Publication(id, year, title, pubkey, jconf, authors, ncitation) self.processer(pub) cursor.close() conn.close () except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) # raise return data
def walk(self): ''' Walk through all persons in database. ''' """ this place put while true inner """ while True: try: lastId = -1 #former while true here # while True: conn = DB.pool().getConnection() cursor = conn.cursor() #print self.sql % (self.gen, start_id, self.limit) cursor.execute(self.sql, (self.update_generation, lastId, self.fetch_size)) data = cursor.fetchall() print "&[Walker]> walk through na_person, %s items" % cursor.rowcount if cursor.rowcount == 0: break id = 0 for id, names, pubcount in data: # fix if self.fix_person_ext and pubcount is None: self.person_update_tool.insertPersonExt(id, self.update_generation, pubcount) lastId = id namelist = names.split(",") for name in namelist: name = name.strip() # Call callback self.processer(Person(id, namelist, pubcount)) cursor.close() conn.close() except MySQLdb.Error, e: #@UndefinedVariable ExceptionHelper.print_exec(e) # raise return data
def walk(self): ''' Walkthrough all pubs in db. ''' try: conn = DB.pool().getConnection() cursor = conn.cursor() if (self.replace_sql): if (self.replace_sql_params): cursor.execute(self.replace_sql, self.replace_sql_params) else: cursor.execute(self.replace_sql) else: # id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation cursor.execute( "select id, year, title, pubkey, jconf, authors, ncitation from publication where %s", self.sql_condition) data = cursor.fetchall() print "walk through %s items" % cursor.rowcount for id, year, title, pubkey, jconf, authors, ncitation in data: pub = Publication(id, year, title, pubkey, jconf, authors, ncitation) self.processer(pub) cursor.close() conn.close() except MySQLdb.DoesNotExist, e: print "Error %d: %s" % (e.args[0], e.args[1]) raise
def getPublicationByConf(self, jconf_id): """Get all publications of a person""" while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( """select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.startpage,\ p.endpage, p.ncitation, p.u_citation_gen from publication p where p.jconf=%s""", jconf_id, ) data = cursor.fetchall() pubs = [] for id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation, gen in data: pub = Publication(id, year, title, pubkey, jconf, authors, ncitation) pub.startpage = startpage pub.endpage = endpage pubs.append(pub) cursor.close() conn.close() return pubs except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) # sys.exit(1) return data
def setPersonUpdateGeneration(self, personId, gen): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("update person_update_ext set u_citation_gen=%s where id=%s", (gen, personId)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1])
def isAllFinished(self, update_generation): while True: conn = DB.pool().getConnection() cursor = conn.cursor() sql = "select \ (select count(*) from na_person) as a, \ (select count(*) from person_update_ext pe where pe.u_citation_gen=%s) as b \ "; cursor.execute(sql, (update_generation)) data = cursor.fetchone() return data[0] == data[1]
def setGeneration(self, newGeneration): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("update constants set value=%s where name=%s", (newGeneration, self.genkey)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1])
def getPersonTotalCount(self): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("select count(*) from na_person") row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: # @UndefinedVariable ExceptionHelper.print_exec(e)
def getTotalCount(self): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("select count(*) from publication") row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def getMinGenerationInDB(self): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("select min(u_citation_gen) from publication") row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def insertPersonExt(self, person_id, update_generation=0, pubcount=0): while True: try: conn = DB.pool().getConnection(); cursor = conn.cursor() cursor.execute("insert into person_update_ext(id,u_citation_gen,pubcount) values(%s,%s,%s)", (person_id, update_generation, 0)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: #@UndefinedVariable ExceptionHelper.print_exec(e)
def getPersonTotalCount(self): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("select count(*) from na_person") row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: #@UndefinedVariable ExceptionHelper.print_exec(e)
def resetPersonPublicationUpdateGen(self, personId): ''' set publication's update_gen to 0 of one person''' while True: try: sql = '''update publication p left join na_author2pub a2p on p.id = a2p.pid set p.u_citation_gen=0 where a2p.aid=%s ''' conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute(sql, (personId)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: #@UndefinedVariable ExceptionHelper.print_exec(e)
def resetPersonPublicationUpdateGen(self, personId): """ set publication's update_gen to 0 of one person""" while True: try: sql = """update publication p left join na_author2pub a2p on p.id = a2p.pid set p.u_citation_gen=0 where a2p.aid=%s """ conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute(sql, (personId)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: # @UndefinedVariable ExceptionHelper.print_exec(e)
def setPersonUpdateGeneration(self, personId, gen): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( "update person_update_ext set u_citation_gen=%s where id=%s", (gen, personId)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: print "Error %d: %s" % (e.args[0], e.args[1])
def markPersonUpdateCitationFinished(self, personId, gen): while True: data = None try: conn = DB.pool().getConnection() cursor = conn.cursor() data = cursor.execute("update person_update_ext set u_citation_gen=%s where id=%s;", (gen, personId)) cursor.close() conn.close() return cursor.rowcount except Exception, e: ExceptionHelper.print_exec(e) # raise return data
def savePerson(self, personId, personName): """Save author2pub""" while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() data = cursor.execute("insert into person (id, fullname) values (%s,%s)", (personId, personName)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: ExceptionHelper.print_exec(e) # raise return data
def getLeftCount(self, generation): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( "select count(*) from publication p \ where p.u_citation_gen is null or p.u_citation_gen < %s", generation) row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def getPersonLeftCount(self, generation): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( "select count(*) from na_person p left join person_update_ext pe on p.id=pe.id \ where pe.u_citation_gen is null or pe.u_citation_gen < %s", generation) row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def getLeftCount(self, generation): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( "select count(*) from publication p \ where p.u_citation_gen is null or p.u_citation_gen < %s", generation, ) row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def markPersonUpdateCitationFinished(self, personId, gen): while True: data = None try: conn = DB.pool().getConnection() cursor = conn.cursor() data = cursor.execute( "update person_update_ext set u_citation_gen=%s where id=%s;", (gen, personId)) cursor.close() conn.close() return cursor.rowcount except Exception, e: ExceptionHelper.print_exec(e) # raise return data
def save(self, person): '''Save author2pub''' while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() data = cursor.execute( "insert into person (id, fullname) values (%s,%s)", (person.id, person.name)) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: ExceptionHelper.print_exec(e) # raise return data
def getPersonLeftCount(self, generation): while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( "select count(*) from na_person p left join person_update_ext pe on p.id=pe.id \ where pe.u_citation_gen is null or pe.u_citation_gen < %s", generation, ) row = cursor.fetchone() cursor.close() conn.close() return row[0] except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def save(self, aid, pid, position): """Save author2pub""" while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() data = cursor.execute( "insert into author2pub (aid,pid,position) values(%s,%s,%s)", (aid, pid, position) ) cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: ExceptionHelper.print_exec(e) # raise return data
def walk(self): ''' Get by pids. ''' while True: try: print "&[Walker]> walk through na_person, BY_ID_LIST: %s items" % len(self.pids) page = 0 count = 300 data = [] conn = DB.pool().getConnection() while True: cursor = conn.cursor() print "Getting People of Page %s" % page # sql temp = [] for item in self.pids[page*count:(page+1)*count]: temp.append(str(item)) if page*count > len(self.pids): break if len(temp) == 0: break inwhere = "".join(["(", ",".join(temp) , ")"]) self.sql = '''select p.id, p.names, pe.id, pe.pubcount from na_person p left join person_update_ext pe on p.id=pe.id \ where (pe.u_citation_gen is null or pe.u_citation_gen < %s) and p.id in %s ''' % (self.update_generation, inwhere) cursor.execute(self.sql) data.extend(cursor.fetchall()) page += 1 if cursor.rowcount == 0: break time.sleep(1) for pid, names, peid, pubcount in data: # fix if self.fix_person_ext and peid is None: self.person_update_tool.insertPersonExt(pid, self.update_generation, pubcount) namelist = names.split(",") for name in namelist: name = name.strip() # Call callback self.processer(Person(pid, namelist, pubcount)) cursor.close() conn.close() except MySQLdb.Error, e: #@UndefinedVariable ExceptionHelper.print_exec(e) return data
def getPersonPubCount(self, personId): '''Get all publication count of a person''' while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( '''select count(*) from publication p left join author2pub a2p on p.id=a2p.pid where a2p.aid=%s''', personId) data = cursor.fetchone() cursor.close() conn.close() return data[0] except MySQLdb.Connection.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) # sys.exit(1) return data
def getPersonPubCount(self, personId): """Get all publication count of a person""" while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( """select count(*) from publication p left join author2pub a2p on p.id=a2p.pid where a2p.aid=%s""", personId, ) data = cursor.fetchone() cursor.close() conn.close() return data[0] except MySQLdb.Connection.Error, e: print "Error %d: %s" % (e.args[0], e.args[1]) # sys.exit(1) return data
def batchUpdate(self, gen, pubs): """batch update pubs, params is ((ncitation, gen, id),...)""" params = [] for pub in pubs: params.append((pub.ncitation, gen, pub.id)) while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("Set AUTOCOMMIT = 0") data = cursor.executemany("update publication set ncitation=%s, u_citation_gen=%s where id=%s", params) conn.commit() cursor.execute("Set AUTOCOMMIT = 1") cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: ExceptionHelper.print_exec(e) # raise return data
def get_author(self, aid, generation): print aid, generation self.sql = """select p.id, p.names, pe.pubcount from na_person p left join person_update_ext pe on p.id=pe.id where (pe.u_citation_gen is null or pe.u_citation_gen < %s) and p.id=%s """ % (generation, aid) try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute(self.sql) data = cursor.fetchall() if cursor.rowcount == 0: return for aid, names, pubcount in data: namelist = [name.strip() for name in names.split(",")] print 'get author: ', namelist return Person(aid, namelist, pubcount) cursor.close() conn.close() except MySQLdb.Error, e: ExceptionHelper.print_exec(e)
def getPublicationByPerson(self, personId, gen): """Get all publications of a person""" while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( """select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.ncitation, p.u_citation_gen from publication p left join na_author2pub a2p on p.id=a2p.pid where (p.u_citation_gen < %s or p.u_citation_gen is null) and a2p.aid=%s""", (gen, personId), ) data = cursor.fetchall() pubs = [] for id, year, title, pubkey, jconf, authors, ncitation, gen in data: pub = Publication(id, year, title, pubkey, jconf, authors, ncitation) pubs.append(pub) cursor.close() conn.close() return pubs except Exception, e: ExceptionHelper.print_exec(e)
def getPublicationByPerson(self, personId, gen): '''Get all publications of a person''' while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute( '''select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.ncitation, p.u_citation_gen from publication p left join na_author2pub a2p on p.id=a2p.pid where (p.u_citation_gen < %s or p.u_citation_gen is null) and a2p.aid=%s''', (gen, personId)) data = cursor.fetchall() pubs = [] for id, year, title, pubkey, jconf, authors, ncitation, gen in data: pub = Publication(id, year, title, pubkey, jconf, authors, ncitation) pubs.append(pub) cursor.close() conn.close() return pubs except Exception, e: ExceptionHelper.print_exec(e)
def batchUpdate(self, gen, pubs): '''batch update pubs, params is ((ncitation, gen, id),...)''' params = [] for pub in pubs: params.append((pub.ncitation, gen, pub.id)) while True: try: conn = DB.pool().getConnection() cursor = conn.cursor() cursor.execute("Set AUTOCOMMIT = 0") data = cursor.executemany( "update publication set ncitation=%s, u_citation_gen=%s where id=%s", params) conn.commit() cursor.execute("Set AUTOCOMMIT = 1") cursor.close() conn.close() return cursor.rowcount except MySQLdb.Error, e: ExceptionHelper.print_exec(e) # raise return data
def walk(self): ''' Walkthrough all persons in db. ''' try: start_id = 0 while True: conn = DB.pool().getConnection() cursor = conn.cursor() #print self.sql % (self.gen, start_id, self.limit) cursor.execute(self.sql, (self.gen, start_id, self.fetch_size)) data = cursor.fetchall() print "&-walker-:> walk person(citation) %s items" % cursor.rowcount if cursor.rowcount == 0: break id = 0 for id, fullname in data: person = Person(id, fullname) self.processer(person) start_id = id cursor.close() conn.close() except MySQLdb.Error, e: #@UndefinedVariable print "Error %d: %s" % (e.args[0], e.args[1]) raise
def walk(self): ''' Walkthrough all persons in db. ''' try: # prepare sql and params sql_template = "%s %s %s" sql_param_sql = "" sql_param_condition = "" sql_param_limit = "" params = [] if self.replace_sql: sql_param_sql = self.replace_sql + " and id > %s" # ugly else: sql_param_sql = "select id, fullname from person where %s %r" if self.sql_condition is not None: sql_param_condition = self.sql_condition if self.replace_sql: sql_param_sql = self.replace_sql + " and id > %s" # ugly # params if self.replace_sql_params is not None: if type(self.replace_sql_params).__name__ == 'list': params.extend(self.replace_sql_params) else: params.append(self.replace_sql_params) # sql_template_param if self.limit > 0: sql_param_limit = "limit %s" params.append(self.limit) sql = sql_template % (sql_param_sql, sql_param_condition, sql_param_limit) print sql sys.exit() conn = DB.pool().getConnection() # /////////////////////// # Arrays.fill(hands, null); # # int count = 0; # long nStartID = 0; # while (true) { # List < T > models = null; # try { # models = getModelsFrom(nStartID); # } catch (SQLException e) { # e.printStackTrace(); # } # if (null == models || models.size() <= 0) { # break; # } # # for (T model : models) { # nStartID = getID(model); # // true logic # foot.process(model); # if (stopWalker) { # System.out.println(" walker stoped."); # return; # } # } # # // throw away # models = null; # # // print # System.out.print("walk through " + count * DATA_FETCH_SIZE + " items.\r"); # count + +; # // if ((count + + +1) % 10 == 0) { # // System.out.print("\t" + count + "\n"); # // } # } # System.out.println("walk throuh complete. " + count * DATA_FETCH_SIZE + " items."); # /////////////////////// cursor = conn.cursor() if(self.replace_sql): if self.replace_sql_params is not None: cursor.execute(self.replace_sql, self.replace_sql_params) else: cursor.execute(self.replace_sql) else: cursor.execute("select id, fullname from person where %s %r", (self.sql_condition, self.limit)) data = cursor.fetchall() print "walk through %s items" % cursor.rowcount for id, fullname in data: person = Person(id, fullname) self.processer(person) cursor.close() conn.close () except MySQLdb.Error, e: #@UndefinedVariable print "Error %d: %s" % (e.args[0], e.args[1]) raise
def test_db_get_one(self): print DB.shortcuts().getOne( "select max(wikicfpid) from conference_events")
def test_db_get_one(self): print DB.shortcuts().getOne("select max(wikicfpid) from conference_events")