コード例 #1
0
ファイル: crawl.py プロジェクト: Lhfcws/ai_hw
def main():
	p = re.compile("http[0-9|:|/| |.|a-z|A-Z]*")
	# connect MYSQL
	res = dbconfig()
	conn = MySQLdb.connect(host=res[0], user=res[1], passwd=res[2])
	conn.select_db('ai_hw')
	cursor = conn.cursor()
	cursor.execute("set names 'utf8'")
	# Init
	M = Main("*****@*****.**", "aiai2012")
	M.start()
	M.login()
	M.clear()

	# Test day
	M.period("2012-09-20", "2012-09-20")

	r = model(cursor)
	key = r[1]
	M.keyword(r[1])
	M.config()

	stop = 0
	while True:
		if stop == 10:
			break
		stop += 1
		if M.end():
			break
		# If we meet a captcha we can solve it by login.
		if M.captcha():
			M.save()
			M.logout()
			M.restart()
			M.login()
			M.load()
	
		stat = M.getUsers()	
		
		# Write the user list into database
		for i in range(len(stat[0])):
			value = [stat[0][i], M.getKeyword(), stat[1][i].encode("utf-8")]
			value[2] = re.sub(p,"",value[2])
			cursor.execute("insert into users value(%s, %s, %s)", value)

		# Flip to next page.
		M.flip()

		conn.commit()

	M.quit()
	value = [M.getKeyword()]
	#cursor.execute("delete from request where keyword='%s'",value);
	#cursor.execute("insert into finish value(%s)", value);
	#conn.commit()
	cursor.close()
	conn.close()
コード例 #2
0
ファイル: plots.py プロジェクト: Lhfcws/ai_hw
def main():
	# connect MYSQL
	res = dbconfig()
	conn = MySQLdb.connect(host=res[0], user=res[1], passwd=res[2])
	conn.select_db('ai_hw')
	cursor = conn.cursor()
	cursor.execute("set names 'utf8'")
	# init 
	M = Main("*****@*****.**", "aiai2012")
	M.start()
	M.login()

	l = model(cursor)
	conn.commit()

	M.keyword(l[1])

	# get Plots
	#plot = trend(cursor, M, date(ls[1], ls[2], ls[3]), date(ls[4], ls[5], ls[6]))
	plot = trend(cursor, M, date(2012,9,20), date(2012,9,30))
	conn.commit()

	cursor.close()
	conn.close()