Esempio n. 1
0
def executarBusca(soup, assunto_atual, pedidoId):

	try:
		conn = psycopg2.connect("dbname='tweet_crawler' user='******' host='104.237.129.216' port='5432' password='******'")
		print "Connected to database"
	except:
		print "I am unable to connect to the database"

	cur=conn.cursor()

	i = capturaNome(soup)
	capturaTweet(soup)
	for tw in tweets:
		print tw
	capturaUsername(soup)
	capturaData(soup)

	j=0
	while i>0:
		datats = datas[i-1]
		horatext,diatext = datats.split("-") 
		datas[i-1] = diatext + ' - ' + horatext		
		dias.insert(j,int(diatext [1:3]))
		tam = len(diatext)
		mes_num = utils.getMes(diatext [tam-11:tam-8])
		meses.insert(j,mes_num)
		#print diatext [7:10]
		anos.insert(j,diatext [tam-4:])

		#print diatext [14:]
		horas.insert(j,horatext)

		j=j+1

		try:

			cur.execute(""" INSERT INTO tweetsold (nome , usuario, tweet, dia, mes, ano, hora, assunto, pedido_id) VALUES  ( %(nome)s,  %(usuario)s, %(tweet)s, %(dia)s, %(mes)s, %(ano)s, %(hora)s, %(assunto)s, %(pedido_id)s)""", ({ "nome" : names[i-1] ,  "usuario" : users[i-1], "tweet" : tweets[i-1], "dia": dias[j-1], "mes": meses[j-1], "ano": anos[j-1], "hora": horas[j-1], "assunto": assunto_atual, "pedido_id": pedidoId }))
			conn.commit()
			print  'tweet: ' + tweets[i-1] + '- data' + datas[i-1]

		except Exception as inst:
			conn.rollback()

		i=i-1
	if conn:
		conn.close()
Esempio n. 2
0
def executarBusca(soup, assunto_atual, pedidoId):

	try:
		conn = psycopg2.connect("dbname='tweet_crawler' user='******' host='104.237.129.216' port='5432' password='******'")
		print "Connected to database"
	except:
		print "I am unable to connect to the database"

	cur=conn.cursor()

	maxNome = capturaNome(soup)
	maxTweet = capturaTweet(soup)
	for tw in tweets:
		print tw
	maxUser = capturaUsername(soup)
	capturaData(soup)

	i = min(maxUser, maxTweet, maxNome)

	filename= os.path.join("logs","crawler" + time.strftime("%d-%m-%Y") + ".log")

	j=0
	index = 0
	valuesString = ""
	while index < i:
		datats = datas[i-1]
		diatext = ""
		horatext = ""
		part1,part2 = datats.split("-") 
		if(len(part1) > len(part2)):
			diatext = part1
			horatext = part2
		else:
			diatext = part2
			horatext = part1 

		#GAMBI DA PORRA DO TWITTER ZUERO

		dias.insert(j,int(diatext [1:3]))
		tam = len(diatext)
		mes_num = utils.getMes(diatext [tam-11:tam-8])
		meses.insert(j,mes_num)
		#print diatext [7:10]
		anos.insert(j,diatext [tam-4:])

		#print diatext [14:]
		horas.insert(j,horatext)
		nameEscaped = adapt(names[index]).getquoted()
		userEscaped = adapt(users[index]).getquoted()
		tweetEscaped = adapt(tweets[index]).getquoted()
		assuntoEscaped = adapt(assunto_atual).getquoted()
		tempStr = "(%s, %s, %s, '%s', '%s', '%s', '%s', %s, '%s')," % \
		(nameEscaped, userEscaped, tweetEscaped, str(dias[j]), str(meses[j]), str(anos[j]), str(horas[j]), assuntoEscaped, pedidoId)
		valuesString = valuesString + tempStr
		j=j+1
		index += 1

	valuesString = valuesString[:len(valuesString)-1]
	try:
		insertSql = "INSERT INTO tweetsold (nome , usuario, tweet, dia, mes, ano, hora, assunto, pedido_id) VALUES " + valuesString
		cur.execute(insertSql)
		conn.commit()
	except Exception as inst:
		conn.rollback()

		
	if conn:
		conn.close()