def executarBusca(soup, assunto_atual, pedidoId):
try:
conn = psycopg2.connect("dbname='tweet_crawler' user='******' host='104.237.129.216' port='5432' password='******'")
print "Connected to database"
except:
print "I am unable to connect to the database"
cur=conn.cursor()
i = capturaNome(soup)
capturaTweet(soup)
for tw in tweets:
print tw
capturaUsername(soup)
capturaData(soup)
j=0
while i>0:
datats = datas[i-1]
horatext,diatext = datats.split("-")
datas[i-1] = diatext + ' - ' + horatext
dias.insert(j,int(diatext [1:3]))
tam = len(diatext)
mes_num = utils.getMes(diatext [tam-11:tam-8])
meses.insert(j,mes_num)
#print diatext [7:10]
anos.insert(j,diatext [tam-4:])
#print diatext [14:]
horas.insert(j,horatext)
j=j+1
try:
cur.execute(""" INSERT INTO tweetsold (nome , usuario, tweet, dia, mes, ano, hora, assunto, pedido_id) VALUES ( %(nome)s, %(usuario)s, %(tweet)s, %(dia)s, %(mes)s, %(ano)s, %(hora)s, %(assunto)s, %(pedido_id)s)""", ({ "nome" : names[i-1] , "usuario" : users[i-1], "tweet" : tweets[i-1], "dia": dias[j-1], "mes": meses[j-1], "ano": anos[j-1], "hora": horas[j-1], "assunto": assunto_atual, "pedido_id": pedidoId }))
conn.commit()
print 'tweet: ' + tweets[i-1] + '- data' + datas[i-1]
except Exception as inst:
conn.rollback()
i=i-1
if conn:
conn.close()
def executarBusca(soup, assunto_atual, pedidoId):
try:
conn = psycopg2.connect("dbname='tweet_crawler' user='******' host='104.237.129.216' port='5432' password='******'")
print "Connected to database"
except:
print "I am unable to connect to the database"
cur=conn.cursor()
maxNome = capturaNome(soup)
maxTweet = capturaTweet(soup)
for tw in tweets:
print tw
maxUser = capturaUsername(soup)
capturaData(soup)
i = min(maxUser, maxTweet, maxNome)
filename= os.path.join("logs","crawler" + time.strftime("%d-%m-%Y") + ".log")
j=0
index = 0
valuesString = ""
while index < i:
datats = datas[i-1]
diatext = ""
horatext = ""
part1,part2 = datats.split("-")
if(len(part1) > len(part2)):
diatext = part1
horatext = part2
else:
diatext = part2
horatext = part1
#GAMBI DA PORRA DO TWITTER ZUERO
dias.insert(j,int(diatext [1:3]))
tam = len(diatext)
mes_num = utils.getMes(diatext [tam-11:tam-8])
meses.insert(j,mes_num)
#print diatext [7:10]
anos.insert(j,diatext [tam-4:])
#print diatext [14:]
horas.insert(j,horatext)
nameEscaped = adapt(names[index]).getquoted()
userEscaped = adapt(users[index]).getquoted()
tweetEscaped = adapt(tweets[index]).getquoted()
assuntoEscaped = adapt(assunto_atual).getquoted()
tempStr = "(%s, %s, %s, '%s', '%s', '%s', '%s', %s, '%s')," % \
(nameEscaped, userEscaped, tweetEscaped, str(dias[j]), str(meses[j]), str(anos[j]), str(horas[j]), assuntoEscaped, pedidoId)
valuesString = valuesString + tempStr
j=j+1
index += 1
valuesString = valuesString[:len(valuesString)-1]
try:
insertSql = "INSERT INTO tweetsold (nome , usuario, tweet, dia, mes, ano, hora, assunto, pedido_id) VALUES " + valuesString
cur.execute(insertSql)
conn.commit()
except Exception as inst:
conn.rollback()
if conn:
conn.close()