def get_worker_information():
information = {}
worker_results = []
information["progress"] = "100%"
information["worker_results"] = worker_results
# calculate total
total = create_pronoun_dictionary()
total['file'] = ""
total_sum = 0
for key, value in result_dict.iteritems():
total[key] += value
total_sum += value
total['total'] = total_sum
# calculate total in percent
total_percent = create_pronoun_dictionary()
total_percent["file"] = ""
total_percent["total"] = 0
for key, value in total.iteritems():
if key == "file":
continue
if total['total'] > 0:
total_percent[key] += round(float(value)/float(total['total']), 2)*100
else:
total_percent[key] = 0
information["total"] = total
information["total_percent"] = total_percent
print information
return information
'''
def main():
tweet_files = get_twitter_file_names()
result_dict = create_pronoun_dictionary()
global result_dict
result_array = []
global result_array
result_array = []
###########################################################################
# Add workers
###########################################################################
num_workers = 4
worker_name = "joakim-lab3-worker-"
def parse_tweets(self, filename):
print "Start parsing tweets"
self.update_state(state='WORKING', meta={'file': None, 'function': "create_pronoun_dictionary()", 'result': None})
pronoun_dictionary = twitter.create_pronoun_dictionary()
# get twitter file
self.update_state(state='WORKING', meta={ 'file': filename, 'function': "get_twitter_file()", 'result': pronoun_dictionary})
# downloading the file and save it as a string
print "downloading ", filename
tweet_file = twitter.get_twitter_file(filename)
print "parsing tweetfile:", filename
# add local dictionary of pronouns to large dictionary of pronouns
self.update_state(state='WORKING', meta={ 'file': filename, 'function': "parse_tweetfile()", 'result': pronoun_dictionary})
pronoun_dictionary = twitter.parse_tweetfile(tweet_file, pronoun_dictionary)
print "Worker done", filename, pronoun_dictionary
return (filename, pronoun_dictionary)
def parse_tweets(self, filename):
print "Start parsing tweets"
self.update_state(state='WORKING',
meta={
'file': None,
'function': "create_pronoun_dictionary()",
'result': None
})
pronoun_dictionary = twitter.create_pronoun_dictionary()
# get twitter file
self.update_state(state='WORKING',
meta={
'file': filename,
'function': "get_twitter_file()",
'result': pronoun_dictionary
})
# downloading the file and save it as a string
print "downloading ", filename
tweet_file = twitter.get_twitter_file(filename)
print "parsing tweetfile:", filename
# add local dictionary of pronouns to large dictionary of pronouns
self.update_state(state='WORKING',
meta={
'file': filename,
'function': "parse_tweetfile()",
'result': pronoun_dictionary
})
pronoun_dictionary = twitter.parse_tweetfile(tweet_file,
pronoun_dictionary)
print "Worker done", filename, pronoun_dictionary
return (filename, pronoun_dictionary)
