-
Notifications
You must be signed in to change notification settings - Fork 0
/
tests.py
344 lines (283 loc) · 11.4 KB
/
tests.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
from os import environ, path
from json import loads, dumps
from uuid import uuid1
from StringIO import StringIO
from tempfile import NamedTemporaryFile
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
from twisted.internet.defer import succeed, Deferred
from twisted.web.client import getPage
from twisted.web import server
from twisted.web.test.test_web import DummyRequest
from twisted.web._responses import *
from twisted.web.static import File
from twisted.web.server import Site
class SmartDummyRequest(DummyRequest):
def __init__(self, method, url, args=None, headers=None, body=None):
DummyRequest.__init__(self, url.split('/'))
self.method = method
self.path =url
self.headers.update(headers or {})
self.content = StringIO()
self.content.write(body or '')
self.content.seek(0)
# set args
args = args or {}
for k, v in args.items():
self.addArg(k, v)
def getAllHeaders(self):
return self.headers
def value(self):
return "".join(self.written)
class DummySite(server.Site):
def get(self, url, args=None, headers=None):
return self._request("GET", url, args, headers)
def post(self, url, args=None, headers=None, body=None):
return self._request("POST", url, args, headers, body=body)
def _request(self, method, url, args, headers, body=None):
request = SmartDummyRequest(method, url, args, headers, body)
resource = self.getResourceFor(request)
result = resource.render(request)
return self._resolveResult(request, result)
def _resolveResult(self, request, result):
if isinstance(result, str):
request.write(result)
request.finish()
return succeed(request)
elif result is server.NOT_DONE_YET:
if request.finished:
return succeed(request)
else:
return request.notifyFinish().addCallback(lambda _: request)
else:
raise ValueError("Unexpected return value: %r" % (result,))
from twisted.trial import unittest
from creepy.api import CreepyAPI
api_url = "http://localhost:8000/"
api_target = api_url + 'index.html'
class WebTest(unittest.TestCase):
def setUp(self):
self.api = CreepyAPI(1, 1)
self.web = DummySite(self.api)
staticfiles = path.join(path.dirname(path.abspath(__file__)), 'static')
files = DummySite(File(staticfiles))
factory = Site(files)
self.static = reactor.listenTCP(8000, factory)
def tearDown(self):
self.static.stopListening()
return self.api.queue.queue.stop()
def assertJson(self, response, data):
resp = loads(response.value())
for key, val in data.items():
self.assertTrue(key in resp)
self.assertEqual(resp[key], val)
def assertCode(self, response, code, data={}):
data['response_code'] = code
self.assertJson(response, data)
@inlineCallbacks
def test_bad_request(self):
response = yield self.web.post(
"/echo", body="{'foo': 1, 'bar': 2",
)
self.assertCode(response, BAD_REQUEST)
@inlineCallbacks
def test_echo(self):
struct = {'foo': 1, 'bar': 2}
response = yield self.web.post(
"/echo", body=dumps(struct),
)
self.assertCode(response, 200, struct)
def test_static(self):
return getPage(api_target)
def job_setup(self, urls=[api_target], depth=0):
struct = {
'urls': urls,
'depth': depth,
}
d = self.web.post(
"/", body=dumps(struct),
)
def handle_job_response(response):
self.assertCode(response, 200)
params = loads(response.value())
jobid = params.get('job')
self.assertIsNotNone(jobid)
self.assertTrue(len(jobid) == 22)
checker = Deferred()
def check_status():
d = self.web.get(
"/status/" + jobid
)
def _check_status(response):
self.assertCode(response, 200)
params = loads(response.value())
status = params.get('status')
if status in ('pending', 'running'):
reactor.callLater(1.0, check_status)
else:
checker.callback(response)
d.addCallback(_check_status)
reactor.callLater(1.0, check_status)
def handle_status_response(response):
self.assertCode(response, 200)
params = loads(response.value())
jobid = params.get('job')
self.assertIsNotNone(jobid)
status = params.get('status')
self.assertTrue(status == 'finished')
_urls = params.get('urls')
self.assertTrue(_urls == urls)
return response
checker.addCallback(handle_status_response)
return checker
d.addCallback(handle_job_response)
# def fail(*args):
# import pdb; pdb.set_trace()
# d.addErrback(fail)
return d
def test_empty_urls(self):
struct = {
'urls': [],
}
d = self.web.post(
"/", body=dumps(struct),
)
def handle_job_response(response):
self.assertCode(response, 400)
d.addCallback(handle_job_response)
return d
def test_depth_0(self):
d = self.job_setup()
def handle_setup_response(response):
params = loads(response.value())
images = params.get('num_images')
self.assertTrue(images == 1, "Images was %s" % images)
d.addCallback(handle_setup_response)
return d
def test_depth_1(self):
d = self.job_setup(depth=1)
def handle_setup_response(response):
params = loads(response.value())
images = params.get('num_images')
self.assertTrue(images == 4, "Images was %s" % images)
d.addCallback(handle_setup_response)
return d
def test_depth_2(self):
d = self.job_setup(depth=2)
def handle_setup_response(response):
params = loads(response.value())
images = params.get('num_images')
self.assertTrue(images == 5)
d.addCallback(handle_setup_response)
return d
def results_test(self, handler, handler_args=None, urls=[api_target], depth=0):
d = self.job_setup(urls, depth=depth)
def handle_setup_response(response):
params = loads(response.value())
jobid = params.get('job')
self.assertIsNotNone(jobid)
d = self.web.get(
"/result/" + jobid,
args=handler_args,
)
d.addCallback(handler)
d.addCallback(handle_setup_response)
return d
# def test_broken(self):
# url = 'http://.com'
# def check_result(response):
# self.assertCode(response,200)
# params = loads(response.value())
# images = params.get('num_images')
# self.assertTrue(images == 0, "Images was %s" % images)
# self.assertTrue(url in params.get('errors', {}))
# return self.results_test(check_result, urls=[url])
def test_default_results(self):
def check_result(response):
self.assertCode(response, 200)
params = loads(response.value())
urls = params.get('urls')
self.assertEqual(urls, [api_target])
num_images = params.get('num_images')
self.assertEqual(num_images, 5)
results = params.get('results')
for image in (
'index.png',
'a.png', 'a2.png',
'b.png', 'c.png'
):
self.assertTrue(api_url+image in results)
return self.results_test(check_result, depth=2)
def test_by_image_results(self):
def check_result(response):
self.assertCode(response, 200)
params = loads(response.value())
urls = params.get('urls')
self.assertEqual(urls, [api_target])
num_images = params.get('num_images')
self.assertEqual(num_images, 5)
results = params.get('results')
self.assertEqual(results, {
api_url + 'index.png': [api_url + 'index.html'],
api_url + 'a.png': [api_url + 'a.html'],
api_url + 'a2.png': [api_url + 'a2.html'],
api_url + 'b.png': [api_url + 'b.html'],
api_url + 'c.png': [api_url + 'c.html'],
})
return self.results_test(check_result, handler_args={
'result_format': 'by_image',
}, depth=2)
def test_by_page_results(self):
def check_result(response):
self.assertCode(response, 200)
params = loads(response.value())
urls = params.get('urls')
self.assertEqual(urls, [api_target])
num_images = params.get('num_images')
self.assertEqual(num_images, 5)
results = params.get('results')
self.assertEqual(results, {
api_url + 'index.html': [api_url + 'index.png'],
api_url + 'a.html': [api_url + 'a.png'],
api_url + 'a2.html': [api_url + 'a2.png'],
api_url + 'b.html': [api_url + 'b.png'],
api_url + 'c.html': [api_url + 'c.png'],
})
return self.results_test(check_result, handler_args={
'result_format': 'by_page',
}, depth=2)
def test_by_page_with_empty_results(self):
def check_result(response):
self.assertCode(response, 200)
params = loads(response.value())
urls = params.get('urls')
self.assertEqual(urls, [api_target])
num_images = params.get('num_images')
self.assertEqual(num_images, 5)
results = params.get('results')
self.assertEqual(results, {
api_url + 'index.html': [api_url + 'index.png'],
api_url + 'a.html': [api_url + 'a.png'],
api_url + 'a2.html': [api_url + 'a2.png'],
api_url + 'b.html': [api_url + 'b.png'],
api_url + 'c.html': [api_url + 'c.png'],
api_url + 'd.html': [],
})
parsed = params.get('num_parsed_pages')
completed = params.get('num_completed')
self.assertTrue(completed == parsed)
return self.results_test(check_result, handler_args={
'result_format': 'by_page',
'include_empty': True,
}, depth=2)
def test_double(self):
def check_result(response):
self.assertCode(response, 200)
params = loads(response.value())
urls = params.get('urls')
self.assertEqual(urls, [api_target, api_target])
num_images = params.get('num_images')
self.assertEqual(num_images, 5)
num_pages = params.get('num_pages')
self.assertTrue(num_pages == 6)
return self.results_test(check_result, urls=[api_target, api_target], depth=2)