コード例 #1
0
	def walk(self):
		''' Walkthrough all persons in db. '''
		while True:
			try:
				lastId = -1
	#			while True:
				conn = DB.pool().getConnection()
				cursor = conn.cursor()
				#print self.sql % (self.gen, start_id, self.limit)
				cursor.execute(self.sql, (self.gen, lastId, self.fetch_size))
				data = cursor.fetchall()
				print "&-walker-:> walk person(citation) %s items" % cursor.rowcount
				if cursor.rowcount == 0:
					break
				id = 0
				for id, fullname, pubcount in data:
					lastId = id
					person = Person(id, fullname, pubcount)
					self.processer(person)
				cursor.close()
				conn.close()
				# sleep 10 minutes next loop
				time.sleep(20)

			except MySQLdb.Error, e: #@UndefinedVariable
				ExceptionHelper.print_exec(e)
#				raise
			return data
コード例 #2
0
    def walk(self):
        ''' Walkthrough all persons in db. '''
        while True:
            try:
                lastId = -1
                #			while True:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                #print self.sql % (self.gen, start_id, self.limit)
                cursor.execute(self.sql, (self.gen, lastId, self.fetch_size))
                data = cursor.fetchall()
                print "&-walker-:> walk person(citation) %s items" % cursor.rowcount
                if cursor.rowcount == 0:
                    break
                id = 0
                for id, fullname, pubcount in data:
                    lastId = id
                    person = Person(id, fullname, pubcount)
                    self.processer(person)
                cursor.close()
                conn.close()
                # sleep 10 minutes next loop
                time.sleep(20)

            except MySQLdb.Error, e:  #@UndefinedVariable
                ExceptionHelper.print_exec(e)
#				raise
            return data
コード例 #3
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def getPublicationByConf(self, jconf_id):
        '''Get all publications of a person'''
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    '''select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.startpage,\ 
					p.endpage, p.ncitation, p.u_citation_gen 
					from publication p 
					where p.jconf=%s''', jconf_id)
                data = cursor.fetchall()
                pubs = []
                for id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation, gen in data:
                    pub = Publication(id, year, title, pubkey, jconf, authors,
                                      ncitation)
                    pub.startpage = startpage
                    pub.endpage = endpage
                    pubs.append(pub)
                cursor.close()
                conn.close()
                return pubs
            except MySQLdb.Error, e:
                print "Error %d: %s" % (e.args[0], e.args[1])
#				sys.exit(1)
            return data
コード例 #4
0
ファイル: start.py プロジェクト: AlexLyj/aminer-spider
	def walk(self):
		''' Walkthrough all pubs in db. '''
		while True:
			try:
				conn = DB.pool().getConnection()
				cursor = conn.cursor()
				if(self.replace_sql):
					if(self.replace_sql_params):
						cursor.execute(self.replace_sql, self.replace_sql_params)
					else:
						cursor.execute(self.replace_sql)
				else:
					# id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation
					cursor.execute("select id, year, title, pubkey, jconf, authors, ncitation from publication where %s", self.sql_condition)
				data = cursor.fetchall()
				print "walk through %s items" % cursor.rowcount
				for id, year, title, pubkey, jconf, authors, ncitation in data:
					pub = Publication(id, year, title, pubkey, jconf, authors, ncitation)
					self.processer(pub)
				cursor.close()
				conn.close ()
			except MySQLdb.Error, e:
				print "Error %d: %s" % (e.args[0], e.args[1])
#				raise
			return data
コード例 #5
0
	def walk(self):
		''' Walk through all persons in database. '''
		""" this place put while true inner """
		while True:
			try:
				lastId = -1
				#former while true here
	#			while True:
				conn = DB.pool().getConnection()
				cursor = conn.cursor()
				#print self.sql % (self.gen, start_id, self.limit)
				cursor.execute(self.sql, (self.update_generation, lastId, self.fetch_size))
				data = cursor.fetchall()
				print "&[Walker]> walk through na_person, %s items" % cursor.rowcount
				if cursor.rowcount == 0:
					break
				id = 0
				for id, names, pubcount in data:
					# fix 
					if self.fix_person_ext and pubcount is None:
						self.person_update_tool.insertPersonExt(id, self.update_generation, pubcount)

					lastId = id
					namelist = names.split(",")
					for name in namelist:
						name = name.strip()
					# Call callback
					self.processer(Person(id, namelist, pubcount))
				cursor.close()
				conn.close()

			except MySQLdb.Error, e: #@UndefinedVariable
				ExceptionHelper.print_exec(e)
#				raise
			return data
コード例 #6
0
 def walk(self):
     ''' Walkthrough all pubs in db. '''
     try:
         conn = DB.pool().getConnection()
         cursor = conn.cursor()
         if (self.replace_sql):
             if (self.replace_sql_params):
                 cursor.execute(self.replace_sql, self.replace_sql_params)
             else:
                 cursor.execute(self.replace_sql)
         else:
             # id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation
             cursor.execute(
                 "select id, year, title, pubkey, jconf, authors, ncitation from publication where %s",
                 self.sql_condition)
         data = cursor.fetchall()
         print "walk through %s items" % cursor.rowcount
         for id, year, title, pubkey, jconf, authors, ncitation in data:
             pub = Publication(id, year, title, pubkey, jconf, authors,
                               ncitation)
             self.processer(pub)
         cursor.close()
         conn.close()
     except MySQLdb.DoesNotExist, e:
         print "Error %d: %s" % (e.args[0], e.args[1])
         raise
コード例 #7
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
    def getPublicationByConf(self, jconf_id):
        """Get all publications of a person"""
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    """select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.startpage,\ 
					p.endpage, p.ncitation, p.u_citation_gen 
					from publication p 
					where p.jconf=%s""",
                    jconf_id,
                )
                data = cursor.fetchall()
                pubs = []
                for id, year, title, pubkey, jconf, authors, startpage, endpage, ncitation, gen in data:
                    pub = Publication(id, year, title, pubkey, jconf, authors, ncitation)
                    pub.startpage = startpage
                    pub.endpage = endpage
                    pubs.append(pub)
                cursor.close()
                conn.close()
                return pubs
            except MySQLdb.Error, e:
                print "Error %d: %s" % (e.args[0], e.args[1])
            # 				sys.exit(1)
            return data
コード例 #8
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def setPersonUpdateGeneration(self, personId, gen):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("update person_update_ext set u_citation_gen=%s where id=%s", (gen, personId))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:
             print "Error %d: %s" % (e.args[0], e.args[1])
コード例 #9
0
	def isAllFinished(self, update_generation):
		while True:
			conn = DB.pool().getConnection()
			cursor = conn.cursor()
			sql = "select \
				(select count(*) from na_person) as a, \
				(select count(*) from person_update_ext pe where pe.u_citation_gen=%s) as b \
			";
			cursor.execute(sql, (update_generation))
			data = cursor.fetchone()
			return data[0] == data[1]
コード例 #10
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def setGeneration(self, newGeneration):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("update constants set value=%s where name=%s", (newGeneration, self.genkey))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:
             print "Error %d: %s" % (e.args[0], e.args[1])
コード例 #11
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def getPersonTotalCount(self):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("select count(*) from na_person")
             row = cursor.fetchone()
             cursor.close()
             conn.close()
             return row[0]
         except MySQLdb.Error, e:  # @UndefinedVariable
             ExceptionHelper.print_exec(e)
コード例 #12
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def getTotalCount(self):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("select count(*) from publication")
             row = cursor.fetchone()
             cursor.close()
             conn.close()
             return row[0]
         except MySQLdb.Error, e:
             ExceptionHelper.print_exec(e)
コード例 #13
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
 def getMinGenerationInDB(self):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("select min(u_citation_gen) from publication")
             row = cursor.fetchone()
             cursor.close()
             conn.close()
             return row[0]
         except MySQLdb.Error, e:
             ExceptionHelper.print_exec(e)
コード例 #14
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
 def getTotalCount(self):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("select count(*) from publication")
             row = cursor.fetchone()
             cursor.close()
             conn.close()
             return row[0]
         except MySQLdb.Error, e:
             ExceptionHelper.print_exec(e)
コード例 #15
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def getMinGenerationInDB(self):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("select min(u_citation_gen) from publication")
             row = cursor.fetchone()
             cursor.close()
             conn.close()
             return row[0]
         except MySQLdb.Error, e:
             ExceptionHelper.print_exec(e)
コード例 #16
0
	def insertPersonExt(self, person_id, update_generation=0, pubcount=0):
		while True:
			try:
				conn = DB.pool().getConnection();
				cursor = conn.cursor()
				cursor.execute("insert into person_update_ext(id,u_citation_gen,pubcount) values(%s,%s,%s)",
							 (person_id, update_generation, 0))
				cursor.close()
				conn.close()
				return cursor.rowcount
			except MySQLdb.Error, e: #@UndefinedVariable
				ExceptionHelper.print_exec(e)
コード例 #17
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
 def setGeneration(self, newGeneration):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("update constants set value=%s where name=%s",
                            (newGeneration, self.genkey))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:
             print "Error %d: %s" % (e.args[0], e.args[1])
コード例 #18
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
 def getPersonTotalCount(self):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute("select count(*) from na_person")
             row = cursor.fetchone()
             cursor.close()
             conn.close()
             return row[0]
         except MySQLdb.Error, e:  #@UndefinedVariable
             ExceptionHelper.print_exec(e)
コード例 #19
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
 def resetPersonPublicationUpdateGen(self, personId):
     ''' set publication's update_gen to 0 of one person'''
     while True:
         try:
             sql = '''update publication p left join na_author2pub a2p on p.id = a2p.pid set p.u_citation_gen=0 where a2p.aid=%s '''
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute(sql, (personId))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:  #@UndefinedVariable
             ExceptionHelper.print_exec(e)
コード例 #20
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def resetPersonPublicationUpdateGen(self, personId):
     """ set publication's update_gen to 0 of one person"""
     while True:
         try:
             sql = """update publication p left join na_author2pub a2p on p.id = a2p.pid set p.u_citation_gen=0 where a2p.aid=%s """
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute(sql, (personId))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:  # @UndefinedVariable
             ExceptionHelper.print_exec(e)
コード例 #21
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
 def setPersonUpdateGeneration(self, personId, gen):
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             cursor.execute(
                 "update person_update_ext set u_citation_gen=%s where id=%s",
                 (gen, personId))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:
             print "Error %d: %s" % (e.args[0], e.args[1])
コード例 #22
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def markPersonUpdateCitationFinished(self, personId, gen):
     while True:
         data = None
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             data = cursor.execute("update person_update_ext set u_citation_gen=%s where id=%s;", (gen, personId))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except Exception, e:
             ExceptionHelper.print_exec(e)
         # 				raise
         return data
コード例 #23
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def savePerson(self, personId, personName):
     """Save author2pub"""
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             data = cursor.execute("insert into person (id, fullname) values (%s,%s)", (personId, personName))
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:
             ExceptionHelper.print_exec(e)
         # 				raise
         return data
コード例 #24
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def getLeftCount(self, generation):
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    "select count(*) from publication p \
					where p.u_citation_gen is null or p.u_citation_gen < %s", generation)
                row = cursor.fetchone()
                cursor.close()
                conn.close()
                return row[0]
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
コード例 #25
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def getPersonLeftCount(self, generation):
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    "select count(*) from na_person p left join person_update_ext pe on p.id=pe.id \
				 where pe.u_citation_gen is null or pe.u_citation_gen < %s", generation)
                row = cursor.fetchone()
                cursor.close()
                conn.close()
                return row[0]
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
コード例 #26
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
    def getLeftCount(self, generation):
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    "select count(*) from publication p \
					where p.u_citation_gen is null or p.u_citation_gen < %s",
                    generation,
                )
                row = cursor.fetchone()
                cursor.close()
                conn.close()
                return row[0]
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
コード例 #27
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def markPersonUpdateCitationFinished(self, personId, gen):
        while True:
            data = None
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                data = cursor.execute(
                    "update person_update_ext set u_citation_gen=%s where id=%s;",
                    (gen, personId))
                cursor.close()
                conn.close()
                return cursor.rowcount
            except Exception, e:
                ExceptionHelper.print_exec(e)
#				raise
            return data
コード例 #28
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def save(self, person):
        '''Save author2pub'''
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                data = cursor.execute(
                    "insert into person (id, fullname) values (%s,%s)",
                    (person.id, person.name))
                cursor.close()
                conn.close()
                return cursor.rowcount
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
#				raise
            return data
コード例 #29
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
    def getPersonLeftCount(self, generation):
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    "select count(*) from na_person p left join person_update_ext pe on p.id=pe.id \
				 where pe.u_citation_gen is null or pe.u_citation_gen < %s",
                    generation,
                )
                row = cursor.fetchone()
                cursor.close()
                conn.close()
                return row[0]
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
コード例 #30
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
 def save(self, aid, pid, position):
     """Save author2pub"""
     while True:
         try:
             conn = DB.pool().getConnection()
             cursor = conn.cursor()
             data = cursor.execute(
                 "insert into author2pub (aid,pid,position) values(%s,%s,%s)", (aid, pid, position)
             )
             cursor.close()
             conn.close()
             return cursor.rowcount
         except MySQLdb.Error, e:
             ExceptionHelper.print_exec(e)
         # 				raise
         return data
コード例 #31
0
	def walk(self):
		''' Get by pids. '''
		while True:
			try:
				print "&[Walker]> walk through na_person, BY_ID_LIST: %s items" % len(self.pids)
				page = 0
				count = 300
				data = []
				conn = DB.pool().getConnection()
				while True:
					cursor = conn.cursor()
					print "Getting People of Page %s" % page
					# sql
					temp = []
					for item in self.pids[page*count:(page+1)*count]:
						temp.append(str(item))
					if page*count > len(self.pids):
						break
					if len(temp) == 0:
						break
					inwhere = "".join(["(", ",".join(temp) , ")"])

					self.sql = '''select p.id, p.names, pe.id, pe.pubcount 
						from na_person p left join person_update_ext pe on p.id=pe.id \
						where (pe.u_citation_gen is null or pe.u_citation_gen < %s) and p.id in %s ''' % (self.update_generation, inwhere)
					cursor.execute(self.sql)
					data.extend(cursor.fetchall())
					page += 1
					if cursor.rowcount == 0:
						break
					time.sleep(1)
						
				for pid, names, peid, pubcount in data:
					# fix 
					if self.fix_person_ext and peid is None:
						self.person_update_tool.insertPersonExt(pid, self.update_generation, pubcount)
					namelist = names.split(",")
					for name in namelist:
						name = name.strip()
					# Call callback
					self.processer(Person(pid, namelist, pubcount))
				cursor.close()
				conn.close()

			except MySQLdb.Error, e: #@UndefinedVariable
				ExceptionHelper.print_exec(e)
			return data
コード例 #32
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def getPersonPubCount(self, personId):
        '''Get all publication count of a person'''
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    '''select count(*) from publication p left join author2pub a2p on p.id=a2p.pid 
					where a2p.aid=%s''', personId)
                data = cursor.fetchone()
                cursor.close()
                conn.close()
                return data[0]
            except MySQLdb.Connection.Error, e:
                print "Error %d: %s" % (e.args[0], e.args[1])
#				sys.exit(1)
            return data
コード例 #33
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
    def getPersonPubCount(self, personId):
        """Get all publication count of a person"""
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    """select count(*) from publication p left join author2pub a2p on p.id=a2p.pid 
					where a2p.aid=%s""",
                    personId,
                )
                data = cursor.fetchone()
                cursor.close()
                conn.close()
                return data[0]
            except MySQLdb.Connection.Error, e:
                print "Error %d: %s" % (e.args[0], e.args[1])
            # 				sys.exit(1)
            return data
コード例 #34
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
    def batchUpdate(self, gen, pubs):
        """batch update pubs, params is ((ncitation, gen, id),...)"""
        params = []
        for pub in pubs:
            params.append((pub.ncitation, gen, pub.id))

        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute("Set AUTOCOMMIT = 0")
                data = cursor.executemany("update publication set ncitation=%s, u_citation_gen=%s where id=%s", params)
                conn.commit()
                cursor.execute("Set AUTOCOMMIT = 1")
                cursor.close()
                conn.close()
                return cursor.rowcount
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
            # 				raise
            return data
コード例 #35
0
 def get_author(self, aid, generation):
     print aid, generation
     self.sql = """select p.id, p.names, pe.pubcount 
         from na_person p left join person_update_ext pe on p.id=pe.id
         where (pe.u_citation_gen is null or pe.u_citation_gen < %s) and p.id=%s
         """ % (generation, aid)
     try:
         conn = DB.pool().getConnection()
         cursor = conn.cursor()
         cursor.execute(self.sql)
         data = cursor.fetchall()
         if cursor.rowcount == 0:
             return
         for aid, names, pubcount in data:
             namelist = [name.strip() for name in names.split(",")]
             print 'get author: ', namelist
             return Person(aid, namelist, pubcount)
         cursor.close()
         conn.close()
     except MySQLdb.Error, e:
         ExceptionHelper.print_exec(e)
コード例 #36
0
 def get_author(self, aid, generation):
     print aid, generation
     self.sql = """select p.id, p.names, pe.pubcount 
         from na_person p left join person_update_ext pe on p.id=pe.id
         where (pe.u_citation_gen is null or pe.u_citation_gen < %s) and p.id=%s
         """ % (generation, aid)
     try:
         conn = DB.pool().getConnection()
         cursor = conn.cursor()
         cursor.execute(self.sql)
         data = cursor.fetchall()
         if cursor.rowcount == 0:
             return
         for aid, names, pubcount in data:
             namelist = [name.strip() for name in names.split(",")]
             print 'get author: ', namelist
             return Person(aid, namelist, pubcount)
         cursor.close()
         conn.close()
     except MySQLdb.Error, e:
         ExceptionHelper.print_exec(e)
コード例 #37
0
ファイル: daos.py プロジェクト: Rygbee/aminer-spider
    def getPublicationByPerson(self, personId, gen):
        """Get all publications of a person"""
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    """select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.ncitation, p.u_citation_gen 
				from publication p left join na_author2pub a2p on p.id=a2p.pid 
				where (p.u_citation_gen < %s or p.u_citation_gen is null) and a2p.aid=%s""",
                    (gen, personId),
                )
                data = cursor.fetchall()
                pubs = []
                for id, year, title, pubkey, jconf, authors, ncitation, gen in data:
                    pub = Publication(id, year, title, pubkey, jconf, authors, ncitation)
                    pubs.append(pub)
                cursor.close()
                conn.close()
                return pubs
            except Exception, e:
                ExceptionHelper.print_exec(e)
コード例 #38
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def getPublicationByPerson(self, personId, gen):
        '''Get all publications of a person'''
        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute(
                    '''select p.id, p.year, p.title, p.pubkey, p.jconf, p.authors, p.ncitation, p.u_citation_gen 
				from publication p left join na_author2pub a2p on p.id=a2p.pid 
				where (p.u_citation_gen < %s or p.u_citation_gen is null) and a2p.aid=%s''',
                    (gen, personId))
                data = cursor.fetchall()
                pubs = []
                for id, year, title, pubkey, jconf, authors, ncitation, gen in data:
                    pub = Publication(id, year, title, pubkey, jconf, authors,
                                      ncitation)
                    pubs.append(pub)
                cursor.close()
                conn.close()
                return pubs
            except Exception, e:
                ExceptionHelper.print_exec(e)
コード例 #39
0
ファイル: daos.py プロジェクト: yinonbaron/aminer-spider
    def batchUpdate(self, gen, pubs):
        '''batch update pubs, params is ((ncitation, gen, id),...)'''
        params = []
        for pub in pubs:
            params.append((pub.ncitation, gen, pub.id))

        while True:
            try:
                conn = DB.pool().getConnection()
                cursor = conn.cursor()
                cursor.execute("Set AUTOCOMMIT = 0")
                data = cursor.executemany(
                    "update publication set ncitation=%s, u_citation_gen=%s where id=%s",
                    params)
                conn.commit()
                cursor.execute("Set AUTOCOMMIT = 1")
                cursor.close()
                conn.close()
                return cursor.rowcount
            except MySQLdb.Error, e:
                ExceptionHelper.print_exec(e)
#				raise
            return data
コード例 #40
0
	def walk(self):
		''' Walkthrough all persons in db. '''
		try:
			start_id = 0
			while True:
				conn = DB.pool().getConnection()
				cursor = conn.cursor()
				#print self.sql % (self.gen, start_id, self.limit)
				cursor.execute(self.sql, (self.gen, start_id, self.fetch_size))
				data = cursor.fetchall()
				print "&-walker-:> walk person(citation) %s items" % cursor.rowcount
				if cursor.rowcount == 0:
					break
				id = 0
				for id, fullname in data:
					person = Person(id, fullname)
					self.processer(person)
				start_id = id
				cursor.close()
				conn.close()

		except MySQLdb.Error, e: #@UndefinedVariable
			print "Error %d: %s" % (e.args[0], e.args[1])
			raise
コード例 #41
0
	def walk(self):
		''' Walkthrough all persons in db. '''
		try:
			# prepare sql and params
			sql_template = "%s %s %s"
			sql_param_sql = ""
			sql_param_condition = ""
			sql_param_limit = ""
			params = []
			if self.replace_sql:
				sql_param_sql = self.replace_sql + " and id > %s" # ugly
			else:
				sql_param_sql = "select id, fullname from person where %s %r"

			if self.sql_condition is not None:
				sql_param_condition = self.sql_condition

			if self.replace_sql:
				sql_param_sql = self.replace_sql + " and id > %s" # ugly

			# params
			if self.replace_sql_params is not None:
				if type(self.replace_sql_params).__name__ == 'list':
					params.extend(self.replace_sql_params)
				else:
					params.append(self.replace_sql_params)

			# sql_template_param
			if self.limit > 0:
				sql_param_limit = "limit %s"
				params.append(self.limit)

			sql = sql_template % (sql_param_sql, sql_param_condition, sql_param_limit)

			print sql
			sys.exit()
			conn = DB.pool().getConnection()

#			///////////////////////
#				Arrays.fill(hands, null);
#
#		int count = 0;
#		long nStartID = 0;
#		while (true) {
#			List < T > models = null;
#			try {
#				models = getModelsFrom(nStartID);
#			} catch (SQLException e) {
#				e.printStackTrace();
#			}
#			if (null == models || models.size() <= 0) {
#				break;
#			}
#
#			for (T model : models) {
#				nStartID = getID(model);
#				// true logic
#				foot.process(model);
#				if (stopWalker) {
#					System.out.println(" walker stoped.");
#					return;
#				}
#			}
#
#			// throw away
#			models = null;
#
#			// print
#			System.out.print("walk through " + count * DATA_FETCH_SIZE + " items.\r");
#			count + +;
#			// if ((count + + +1) % 10 == 0) {
#			// System.out.print("\t" + count + "\n");
#			// }
#		}
#		System.out.println("walk throuh complete. " + count * DATA_FETCH_SIZE + " items.");
#			///////////////////////



			cursor = conn.cursor()

			if(self.replace_sql):
				if self.replace_sql_params is not None:
					cursor.execute(self.replace_sql, self.replace_sql_params)
				else:
					cursor.execute(self.replace_sql)
			else:
				cursor.execute("select id, fullname from person where %s %r", (self.sql_condition, self.limit))

			data = cursor.fetchall()
			print "walk through %s items" % cursor.rowcount
			for id, fullname in data:
				person = Person(id, fullname)
				self.processer(person)
			cursor.close()
			conn.close ()
		except MySQLdb.Error, e: #@UndefinedVariable
			print "Error %d: %s" % (e.args[0], e.args[1])
			raise
コード例 #42
0
 def test_db_get_one(self):
     print DB.shortcuts().getOne(
         "select max(wikicfpid) from conference_events")
コード例 #43
0
ファイル: test.py プロジェクト: AlexLyj/aminer-spider
	def test_db_get_one(self):
		print DB.shortcuts().getOne("select max(wikicfpid) from conference_events")