-
Notifications
You must be signed in to change notification settings - Fork 1
/
tasks.py
29 lines (19 loc) · 1.08 KB
/
tasks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
from celery import Celery
import twitterparser as twitter
app = Celery('tasks', backend='amqp', broker='amqp://antus:antusantus@130.238.29.29/vhost_antus')
@app.task(bind=True)
def parse_tweets(self, filename):
print "Start parsing tweets"
self.update_state(state='WORKING', meta={'file': None, 'function': "create_pronoun_dictionary()", 'result': None})
pronoun_dictionary = twitter.create_pronoun_dictionary()
# get twitter file
self.update_state(state='WORKING', meta={ 'file': filename, 'function': "get_twitter_file()", 'result': pronoun_dictionary})
# downloading the file and save it as a string
print "downloading ", filename
tweet_file = twitter.get_twitter_file(filename)
print "parsing tweetfile:", filename
# add local dictionary of pronouns to large dictionary of pronouns
self.update_state(state='WORKING', meta={ 'file': filename, 'function': "parse_tweetfile()", 'result': pronoun_dictionary})
pronoun_dictionary = twitter.parse_tweetfile(tweet_file, pronoun_dictionary)
print "Worker done", filename, pronoun_dictionary
return (filename, pronoun_dictionary)