-
Notifications
You must be signed in to change notification settings - Fork 1
/
backend.py
284 lines (242 loc) · 10.2 KB
/
backend.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
# from word2vec import Word2Vec
import dateutil.parser
from gzip import GzipFile
import simplejson as json
import numpy as np
from tuplenastics import *
import text_nltk
from gzip import GzipFile
import pprint
from redis import Redis
import scipy.stats
import spark
import sys
import text_nltk
def redis():
try:
return redis.redis
except AttributeError:
redis.redis = Redis()
return redis.redis
def mean_headline(x):
if 'headline' not in x or 'main' not in x['headline'] or not x['headline']['main']:
return None
if 'lead_paragraph' not in x or not x['lead_paragraph']:
return None
words = text_nltk.lemma_tokenize(x['headline']['main'])
if len(words) < 5:
return None
words += text_nltk.lemma_tokenize(x['lead_paragraph'])
return np.nanmean([text_nltk.vectors(w) for w in words], axis=0)
def max_emotion(x):
return sorted(x.iteritems(), key=lambda x: -x[1])[0][0]
def divide(x):
return x / x[0]
'''
prepend key - k_(f)
apply f to key - fk_(f)
apply f to value - _fv(f)
drop key, v - k v
swap key, value - swap
lambda x: x[k] if k in x else None - nk_ _nv nx
(lambda x: x[k] if k in x else None)
'''
if False:
test = sc.parallelize([
'{"query": "John Doe", "pub_date": "today", "headline": {"main": "John is pleased today"}}',
'{}',
'{"query": "John Biggs", "pub_date": "today", "headline": {"main": "John is delighted today"}}'
]).flatMap(flat_json)
assert test.count() == 3, "count != 3"
assert test.take(1)[0]['query'] == "John Doe", "query not parsed"
emotions = test.flatMap(f_skip(mean_headline)).map(
empathy).map(max_emotion).collect()
print emotions
assert emotions[0] == 'happy', 'John Doe is not pleased: %s' % emotions[0]
assert emotions[
1] == 'happy', 'John Doe is not delighted: %s' % emotions[1]
happiness = test.flatMap(f_skip(mean_headline)).map(
empathy).map(lambda x: x['happy']).collect()
print happiness
assert happiness[0] >= happiness[
1], 'pleased < delighted: %f %f' % happiness
'''
print test \
.flatMap(k_skip(lambda x: x['query'] if 'query' in x else None)) \
.flatMap(_fv_skip(mean_headline)) \
.map(_fv(empathy)) \
.map(_fv(lambda x: x['happy'])) \
.collect() #map(k_(empathy)).map(k_(max_emotion)).collect()
'''
articles = test.flatMap(
add_skip(lambda x: (x['query'], x['pub_date']) if 'query' in x else None))
scores = articles \
.flatMap(_fv_skip(mean_headline)) \
.map(_fv(lambda x: np.dot(x, text_nltk.vectors('happy'))))
join = articles \
.join(scores) \
.map(fv(lambda x: dict_kv(x[0], 'score', x[1]))) \
.map(add_(lambda x: x['score'])).sortByKey(True).map(v).collect()
assert join[0][
'query'] == 'John Biggs', 'John Biggs is not the least happy after join.'
print >>sys.stderr, 'TEST OK'
'''
.join(test2) \
.map(v(lambda x: dict_kv(x[1], 'score', x[0]))) \
.map(lambda x: dict_kv(x, 'image', '/static/images/%s.png' % x['query'])) \
'''
# define the text data RDD and the dictionary RDD
#data = sc.textFile("s3n://insight-data-oregon/data/nyt2/", 20).map(json.loads)
#data = sc.textFile("file:///mnt/data/1403309348.json.gz").parallelize(40).map(json.loads)
#data = sc.parallelize(GzipFile('/mnt/data/1403309348.json.gz'), 10).map(json.loads)
#data = sc.textFile('s3n://insight-data-oregon/20140625_nyt.json.gz').repartition(10).flatMap(flat_json)
'''
data = {}
data['brand'] = sc.textFile('s3n://insight-data-oregon/20140627_business_brand_nyt.json.gz').repartition(39).flatMap(flat_json).cache()
data['celebrity'] = sc.textFile("file:///mnt/data/1403309348.json.gz").repartition(40).flatMap(flat_json).cache()
#data = sc.textFile('s3n://insight-data-oregon/business_brand_nyt2.json', 10).repartition(10).map(json.loads)
#data = data.sample(False, 0.1).cache()
print '\n\nPartitions: %i\n\n' % data['brand']._jrdd.splits().size()
print '\n\nPartitions: %i\n\n' % data['celebrity']._jrdd.splits().size()
article_index = {}
article_vector = {}
identity_vector = {}
for k in data:
article_index[k] = data[k].flatMap(add_skip(lambda x: (x['query'], x['pub_date'])))
article_vector[k] = article_index[k].flatMap(_fv_skip(mean_headline))
identity_vector[k] = article_vector[k] \
.map(fk_(lambda x: x[0])) \
.reduceByKey(lambda a, b: a+b) \
.filter(lambda x: x[1][0] >= 3) \
.map(_fv(divide)).cache()
'''
def check_headline(x):
# check full name
tf_query = x['query'] in x['headline']['main']
if 'seo' in x['headline']:
tf_query = tf_query or x['query'] in x['headline']['seo']
if 'print_headline' in x['headline']:
tf_query = tf_query or x['query'] in x['headline']['print_headline']
# check last name
last_ind = x['query'].rfind(' ')
if last_ind > 0:
last_name = x['query'][(last_ind + 1):]
tf_query = tf_query or last_name in x['headline']['main']
if 'seo' in x['headline']:
tf_query = tf_query or last_name in x['headline']['seo']
if 'print_headline' in x['headline']:
tf_query = tf_query or last_name in x['headline']['print_headline']
return tf_query
# filter those who does not appear in the title
#data = data.filter(check_headline)
# print 'data count after headline filtering'
#dic = sc.textFile("s3n://insight-data-oregon/vectors_25d.txt.gz").map(lambda x: x.split(' ')).map(lambda x: (x[0], [1.0,]+map(float,x[1:])))
def x_text(x):
return x['headline']['main'] if 'headline' in x and 'main' in x['headline']['main'] and x['headline']['main'] else None
# pick lead paragraph - tokenize and lammentize
'''
print data.take(1)
article_index = data.flatMap(add_skip(lambda x: (x['query'], x['pub_date'])))
article_vector = article_index.flatMap(_fv_skip(mean_headline))
identity_score = article_vector \
.map(fk_(lambda x: x[0])) \
.reduceByKey(lambda a, b: a+b) \
.map(_fv(divide)) \
.map(_fv(empathy))
'''
result_format = lambda x: {
'name': x[1], 'score': x[0], 'image': '/static/images/%s.png' % x[1]}
#result_format = lambda x: {'name': x[1][0], 'score': x[0] if not np.isnan(x[0]) else 0, 'image': '/static/images/%s.png' % x[1][0], 'time': x[1][1]}
'''
emotion = 'happy'
text = json.dumps(article_vector.map(empathy).take(1)) #map(lambda x: (x[1][emotion], x[0])).filter(lambda x: not np.isnan(x[0])).sortByKey(False).map(result_format).take(36))
print text
sys.exit()
'''
def confusion(model=None):
emotions = ['joy', 'sadness', 'disgust', 'anger', 'surprise', 'fear']
data = spark.data('semeval', cores=16) \
.map(lambda x: spark.dict_kv(x,
'emotion',
text_nltk.empathy(
text_nltk.mean_vector(
x['text'], model=model),
model=model
)
)).collect()
sample = data[0]
print >>sys.stderr, sample
results = []
for emotion in emotions:
for prediction in sample['emotion'].keys():
x = np.asarray([float(row[emotion]) for row in data])
y = np.asarray([row['emotion'][prediction] for row in data])
ind = np.where((np.isnan(x) + np.isnan(y)) == 0)
z = (emotion, prediction, ) + scipy.stats.pearsonr(x[ind], y[ind])
results.append(
{'emotion': z[0], 'vector': z[1], 'r': z[2], 'p': z[3]})
return results
def add_emotion(a, b):
if 'n' not in a:
a['n'] = 1
if 'n' not in b:
b['n'] = 1
for k in a:
a[k] = a[k] + (b[k] if k in b else 0)
return a
def div_emotion(a):
n = a['n'] if 'n' in a else 1
for k in a:
a[k] = a[k] / n
return a
def all_identities(source, model=None):
if not model:
model = 'vectors_25d'
return spark.data(source, cores=4) \
.map(lambda x: ((x['query'], '%s-01' % x['pub_date'][:7]), text_nltk.empathy(x['headline_main_%s' % model], model=model))) \
.reduceByKey(add_emotion).map(lambda x: {'query': x[0][0], 'pub_date': x[0][1], 'emotion': div_emotion(x[1])}).collect()
def top_articles(source, emotion):
return article_vector[source] \
.map(_fv(lambda x: np.dot(text_nltk.vectors(emotion), x))) \
.map(fk_(lambda k: k[1])) \
.map(swap).sortByKey(False) \
.map(result_format).take(36)
def all_articles(source, model=None):
if not model:
model = 'vectors_25d'
results = spark.data(source, cores=16).map(lambda x: dict_kv(
x, 'emotion', text_nltk.empathy(x['headline_main_%s' % model], model=model)))
return results.collect()
def submit_response(key, data):
redis().set(key, json.dumps(data, ignore_nan=True))
redis().publish('response', key)
def listen_redis():
pubsub = redis().pubsub()
pubsub.subscribe('command')
for item in pubsub.listen():
print item
if item['type'] == 'message':
params = redis().hgetall(item['data'])
key = params['__key']
del params['__key']
print >>sys.stderr, 'KEY: %s' % key
if key.endswith('all_articles.json'):
submit_response(key, all_articles(params['source']))
continue
if 'all_identities.json' in key:
submit_response(key, all_identities(
params['source'], model=params['model'] if 'model' in params else None))
continue
if key.endswith('confusion.json'):
submit_response(
key, confusion(model=params['model'] if 'model' in params else None))
continue
if key.endswith('top.json'):
submit_response(
key, top_json(params['source'], params['emotion'], params['identity']))
continue
print key
print params
#submit_response(key, params)
if __name__ == '__main__':
listen_redis()