def mining(self, stamp_start, stamp_finish):
stamp_start, stamp_finish = int(stamp_start) + 1, int(stamp_finish)
cont = 5
pivo = 5
options = webdriver.ChromeOptions()
options.add_argument('headless')
self._driver = webdriver.Chrome()
while stamp_start < stamp_finish:
url = "https://twitter.com/search?f=tweets&vertical=default&q=%23{}%20since%3A{}%20until%3A{}&l=pt&src=typd".format(
self._hastag, stamp_start, stamp_start + self.STEPP_TIMESTAMP)
driver = self._driver
driver.get(url)
assert "since" in driver.title
elementList = driver.find_elements_by_class_name("js-stream-tweet")
lasttSizeList = len(elementList)
while True:
body = driver.find_element_by_tag_name('body')
body.send_keys(Keys.END)
time.sleep(2)
elementList = driver.find_elements_by_class_name(
"js-stream-tweet")
sizeList = len(elementList)
if lasttSizeList == sizeList:
break
elif sizeList > 100:
break
lasttSizeList = len(elementList)
stringList = []
for tweet in reversed(elementList):
idTweet = tweet.get_attribute("data-tweet-id")
tweetTimeStamp = tweet.find_elements_by_class_name(
"js-short-timestamp")[0].get_attribute("data-time")
stringList.append(idTweet + ' ' + tweetTimeStamp + '\n')
dao = Dao()
dao.insert(
'manager',
['hastag', 'idTweet', 'idCandidato', 'timeStamp'],
[self._hastag, idTweet, self._candidato, tweetTimeStamp])
print('...')
cont += 1
if cont > pivo:
pivo += 5
self.get_status(stamp_start)
stamp_start = stamp_start + self.STEPP_TIMESTAMP + 1
self._driver.close()
self._driver.quit()
class _Repository:
def __init__(self):
self._conn = sqlite3.connect('database.db')
self.vaccines = Dao(DTO.Vaccine, self._conn)
self.suppliers = Dao(DTO.Supplier, self._conn)
self.clinics = Dao(DTO.Clinic, self._conn)
self.logistics = Dao(DTO.Logistic, self._conn)
def close(self):
self._conn.commit()
self._conn.close()
def create_tables(self):
self._conn.executescript("""
CREATE TABLE IF NOT EXISTS logistics (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
count_sent INTEGER NOT NULL ,
count_received INTEGER NOT NULL
);
CREATE TABLE IF NOT EXISTS suppliers (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
logistic INTEGER REFERENCES logistics(id)
);
CREATE TABLE IF NOT EXISTS clinics (
id INTEGER PRIMARY KEY,
location TEXT NOT NULL,
demand INTEGER NOT NULL ,
logistic INTEGER REFERENCES logistics(id)
);
CREATE TABLE IF NOT EXISTS vaccines (
id INTEGER PRIMARY KEY,
date DATE NOT NULL,
supplier INTEGER REFERENCES suppliers(id),
quantity INTEGER NOT NULL
);
""")
def receiveShipment(self, nameOfSup, amount, date):
# insert the next vaccine to the vaccine table
# get the id of the logistics from the suppliers table using the name
supplier = self.suppliers.find(name=nameOfSup)
supplierIndex = supplier[0].id
# get the id of the last inserted line to create a new id for the new vaccine
lastId = self.vaccines.getLastInsertedId()
newId = lastId[0] + 1
newVaccine = DTO.Vaccine(newId, date, supplierIndex, amount)
self.vaccines.insert(newVaccine)
idOfLogistics = supplier[0].logistic
# update the count_received of this logistics company in logistics table
logistic = self.logistics.find(id=idOfLogistics)
currCountRec = logistic[0].count_Received
set_value = {'count_received': currCountRec + int(amount)}
# only where the id = idOfLogistics we got from the find query
cond = {'id': idOfLogistics}
self.logistics.update(set_value, cond)
def sendShipment(self, locationOfClinic, amount):
clinic = self.clinics.find(location=locationOfClinic)
# get the id of the logistic of this clinic
idOfLogistics = clinic[0].logistic
# update the count_sent of this logistics company in logistics table
logistic = self.logistics.find(id=idOfLogistics)
currCountSent = logistic[0].count_Sent
set_value = {'count_sent': currCountSent + int(amount)}
# only where the id = idOfLogistics we got from the find query
cond = {"id": idOfLogistics}
self.logistics.update(set_value, cond)
# remove amount from inventory
allVaccines = self.vaccines.findWithASCOrder('date')
tempAmount = int(amount)
for vaccine in allVaccines:
if tempAmount == 0:
break
# we need to delete the line since the quantity will be zero
if vaccine.quantity <= int(tempAmount):
self.vaccines.delete(id=vaccine.id)
tempAmount = tempAmount - int(vaccine.quantity)
# if we can take amount and not delete
else:
set_value = {'quantity': vaccine.quantity - int(tempAmount)}
cond = {"id": vaccine.id}
self.vaccines.update(set_value, cond)
tempAmount = 0
# remove amount from the demand of location
currDemand = clinic[0].demand
set_value = {"demand": currDemand - int(amount)}
cond = {"location": locationOfClinic}
self.clinics.update(set_value, cond)
import sys
from extract_tweet_ids import HastagMining
import time
from tweet_from_id import TweetFromID
from models import Tweet,User
from sentiment_analyze import Linguakit
from utils import printError, limparTexto
s1 = sys.argv[1]
if s1 == 'newHashtag':
candidato, hastag = sys.argv[2], sys.argv[3]
dao = Dao()
r = dao.insert('manager', ['hastag', 'idTweet', 'idCandidato', 'timeStamp'],[hastag, '0', candidato, 0])
print(">>> new hastag saved for mining use 'main.py mineHashtag {} {} {}'".format(candidato, hastag, 'p'))
elif s1 == 'mineHashtag':
candidato, hastag = sys.argv[2], sys.argv[3]
p = False
if sys.argv[4] == 'p':
p = True
mining = HastagMining(hastag, candidato, print_status = p)
mining.start()
if s1 =='mineTweet':
arq = open('Last_id.txt','r')
lastId = arq.read()