def main():
# Read config file
parser = ConfigParser()
parser.read('../config.ini')
sections = ["Default", "Seeds"]
driver_path = parser.get(sections[0], "driverpath")
baseurl = parser.get(sections[0], "baseurl")
num_tags = parser.get(sections[0], "numtags")
related_tag_limit = parser.get(sections[0], "relatedtaglimit")
num_seeds = parser.get(sections[0], "numseeds")
# get seed hash tags
seeds = []
for i in range(int(num_seeds)):
seeds.append(parser.get(sections[1], "seed{:d}".format(i + 1)))
add_hash_symbol(seeds)
# browser object and hashtag objects
browser = BrowserController(driver_path)
browser.browser_open()
database = dict()
for seed in seeds:
hashtags = Hashtag(seed, baseurl, int(num_tags), int(related_tag_limit))
database.update(hashtags.scrapping_loop(browser))
browser.browser_close()
# Print collected hashtags
print_sorted_database(database)
def process(self, tuple):
tweet_id = tuple.values["tweet-id"]
sentiment = tuple.values["sentiment"]
country = tuple.values["country"]
hashtags = tuple.values["hashtags"]
storm.logInfo("Received tweet with tweet id: "+str(tweet_id))
for h in hashtags:
try:
db_obj = Hashtag.get(hashtag=h)
db_obj_count = db_obj.count
db_obj_overall_sentiment = (((db_obj.overall_sentiment * db_obj_count) + sentiment) / (db_obj.count + 1))
db_obj_country_sentiment = db_obj.country_sentiment["country"]
db_obj_country_sentiment = (((db_obj_country_sentiment["sentiment"] * db_obj_country_sentiment["count"]) +
sentiment) / (db_obj.country_sentiment["count"] + 1))
db_obj_country_sentiment["count"] += 1
db_obj.count += 1
db_obj.overall_sentiment = db_obj_overall_sentiment
db_obj.country_sentiment = db_obj_country_sentiment
db_obj.save()
storm.logInfo("Updating Hashtag: "+h)
except:
db_obj = Hashtag(hashtag=h, overall_sentiment=sentiment, country_sentiment={country: {
"sentiment": sentiment, "count": 1}}, count=1)
db_obj.save()
storm.logInfo("Inserting New Hashtag: "+h)
def publicarPost(self, nick, id_hashtag, contingut_post):
"""
Metode que registra posts a partir d'un usuari i d'un hashtag, el Doctest no llista usuaris ja que el timestamp no sera mai igual
>>> i = iTicApp()
>>> i.publicarPost('Bernat', 'munta', 'bon diaaa' )
Usuari no creat
"""
if nick not in list(self.__usuaris):
print("Usuari no creat")
else:
p = Posts(contingut_post, id_hashtag)
p.registraUsuari(nick)
if (id_hashtag not in self.__hashtags.keys()):
print("Hashtag no creat")
self.__hashtags[id_hashtag] = Hashtag(id_hashtag)
p.registraHashtag(Hashtag(id_hashtag))
self.__hashtags[id_hashtag].hashTagUtilitzat()
else:
self.__hashtags[id_hashtag].hashTagUtilitzat()
self.__usuaris[nick].registraPost(p)
if (nick not in list(self.__posts)):
self.__posts[nick] = []
self.__posts[nick].append(p)
else:
self.__posts[nick].append(p)
def hashtag_add_page():
if request.method == 'GET':
return render_template('hashtag_add.html')
else:
name = str(request.form['name'])
hashtag = Hashtag(name)
current_app.hashtags.add_hashtag(hashtag)
hashtag_id = current_app.hashtags.get_hashtag(hashtag.name)
return redirect(url_for('hashtags.hashtags_page'))
def add_tokens_to_list(sentence, filename, common_tokens):
common_words = []
for token in common_tokens:
word = " " + token.lower() + " "
phrase = " " + sentence.lower() + " "
if word in phrase:
common_words.append(token)
for common_word in common_words:
hashtag = Hashtag(filename, common_word, sentence)
data.append(hashtag)
def get_hashtags(self):
with dbapi2.connect(app.config['dsn']) as connection:
cursor = connection.cursor()
statement = """SELECT ID, NAME FROM HASHTAGS ORDER BY ID"""
cursor.execute(statement)
hashtagTable = [(id, Hashtag(name))
for id, name in cursor]
connection.commit()
cursor.close()
return hashtagTable
def process(self, tuple):
tweet_id = tuple.values["tweet-id"]
sentiment = tuple.values["sentiment"]
country = tuple.values["country"]
hashtags = tuple.values["hashtags"]
storm.logInfo("Received tweet with tweet id: " + str(tweet_id))
for h in hashtags:
try:
db_obj = Hashtag.get(hashtag=h)
db_obj_count = db_obj.count
db_obj_overall_sentiment = ((
(db_obj.overall_sentiment * db_obj_count) + sentiment) /
(db_obj.count + 1))
db_obj_country_sentiment = db_obj.country_sentiment["country"]
db_obj_country_sentiment = (
((db_obj_country_sentiment["sentiment"] *
db_obj_country_sentiment["count"]) + sentiment) /
(db_obj.country_sentiment["count"] + 1))
db_obj_country_sentiment["count"] += 1
db_obj.count += 1
db_obj.overall_sentiment = db_obj_overall_sentiment
db_obj.country_sentiment = db_obj_country_sentiment
db_obj.save()
storm.logInfo("Updating Hashtag: " + h)
except:
db_obj = Hashtag(hashtag=h,
overall_sentiment=sentiment,
country_sentiment={
country: {
"sentiment": sentiment,
"count": 1
}
},
count=1)
db_obj.save()
storm.logInfo("Inserting New Hashtag: " + h)
def afegeixHashtag(self, id):
"""
S'afeig el Hashtag al diccionari corresponent, no es fa servir mai
>>> i = iTicApp()
>>> i.afegeixHashtag('munta')
>>> i.afegeixHashtag('munta')
Hashtag ja existent
"""
if id in self.__hashtags.keys():
print("Hashtag ja existent")
else:
h = Hashtag(id)
self.__hashtags[id] = h
def getRelatedHashtags(tag):
headers = {
"user-agent": constants.USER_AGENT,
"api-token": buildApiToken(tag)
}
req = requests.get(constants.GENERATOR_URL.format(tag), headers=headers)
jsonResponse = req.json()
relatedHashtags = []
for h in jsonResponse["results"]:
relatedHashtags.append(Hashtag(h["tag"]))
return relatedHashtags
def readHashtag(id, pathOfText):
hashtagCount = 0
numberOfHashtag = len(pathOfText.find_elements_by_css_selector("a"))
while(hashtagCount < numberOfHashtag):
possibleHashtag = pathOfText.find_elements_by_css_selector("a")[hashtagCount].text
if "#" in possibleHashtag:
hashtag = possibleHashtag.replace("#","")
h = Hashtag(id, hashtag)
hashtagInJVersion = json.dumps(h.__dict__)
print hashtagInJVersion
if(serverOn):
connection = httplib.HTTPConnection(host, port)
connection.request("POST","/api/ig/hashtags/", hashtagInJVersion, headers)
response = connection.getresponse()
print("Status: {} and reason: {}".format(response.status, response.reason))
hashtagCount += 1
def get_hashtagObj(self, hashtag_name):
hashtag = Hashtag(hashtag_name)
return hashtag
def getHashtag(self):
"""
"""
return Hashtag(self.hashtag)
def registraHashtag(self, id):
"""
"""
self.hashtag = Hashtag(id)
from hashtag import Hashtag
from methods import get_data
myHashtags = ('lodz')
for element in myHashtags:
newHashtag = Hashtag(element)
request = get_data(newHashtag)
# result = put_dictionary(newHashtag)
# print_dictionary(result)