/
isitup.py
166 lines (133 loc) · 4.05 KB
/
isitup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
import json
import urllib
import urllib2
from functools import partial
from multiprocessing.dummy import Pool
import uwsgi
from bottle import get, default_app, request, abort, response, route
EXPIRE = 60
USERAGENT = "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)"
try:
LOCKS = int(uwsgi.opt['locks'])
except:
print '"uwsgi --locks N" is mandatory'
exit(1)
def invalid():
response.status = 400
return 'URL is down'
class lock(object):
def __init__(self, lock_idx):
self.lock_idx = lock_idx
def __enter__(self, *args):
uwsgi.lock(self.lock_idx)
def __exit__(self, *args):
uwsgi.unlock(self.lock_idx)
@get('/')
def index():
try:
url = request.params['url']
except KeyError:
abort(400, 'Please provide a url')
return # only for pylint
url_hash = hash(url)
cache_key = str(url_hash)
lock_idx = url_hash % LOCKS
with lock(lock_idx):
cache = uwsgi.cache_get(cache_key)
if cache == 'd':
return invalid()
if cache == 'u':
return url
try:
req = urllib2.Request(
url,
headers={'User-Agent': USERAGENT}
)
urllib2.urlopen(req, timeout=10)
except:
uwsgi.cache_set(cache_key, 'd', EXPIRE)
return invalid()
else:
uwsgi.cache_set(cache_key, 'u', EXPIRE)
return url
# reconciliation
METADATA = {
"name": "IsItUp? Reconciliation Service",
"defaultTypes": [],
}
def search(host, query):
# Example match format:
"""
match = {
"id": 'term id',
"name": 'term label',
"score": 50,
"match": False, # Return True for exact match.
"type": [{"id": "/base/oceanography/research_cruise",
"name": "Research Cruise"}]
}
matches.append(match)
"""
# print 'SEARCH ON {} USING QUERY {}'.format(host, query)
url = 'http://{}/?{}'.format(host, urllib.urlencode(
{'url': query.encode('utf8')})
)
try:
urllib2.urlopen(url)
except Exception, e:
return []
else:
return [{
'id': query,
'name': query,
'match': True,
'score': 100,
'type': [{
'id': '/',
'name': 'Basic service',
}]
}]
def jsonpify(obj):
"""
Like jsonify but wraps result in a JSONP callback if a 'callback'
query param is supplied.
"""
try:
callback = request.params['callback']
except KeyError:
content = json.dumps(obj)
else:
content = '{}({})'.format(callback, json.dumps(obj))
response.content_type = "text/javascript"
return content
@route('/reconcile', method=['GET', 'POST'])
def reconcile():
host = request.headers['Host']
# If a single 'query' is provided do a straightforward search.
query = request.params.get('query')
if query:
# If the 'query' param starts with a "{" then it is a JSON object
# with the search string as the 'query' member. Otherwise,
# the 'query' param is the search string itself.
if query.startswith("{"):
query = json.loads(query)['query']
results = search(host, query)
return jsonpify({"result": results})
# If a 'queries' parameter is supplied then it is a dictionary
# of (key, query) pairs representing a batch of queries. We
# should return a dictionary of (key, results) pairs.
queries = request.params.get('queries')
# print queries
if queries:
queries = json.loads(queries)
def f(host, t):
key, query = t
return key, {"result": search(host, query['query'])}
pool = Pool(10)
f = partial(f, host)
results = dict(pool.map(f, queries.items()))
return jsonpify(results)
# If neither a 'query' nor 'queries' parameter is supplied then
# we should return the service metadata.
return jsonpify(METADATA)
application = default_app()