def test_post(self):
project_name = 'test_project'
post_data = {}
post_data['egg'] = open(
path.join(path.dirname(__file__), '..',
'test_project-1.0-py2.7.egg'), 'rb')
post_data['project'] = project_name
post_data['version'] = '1.0'
post_data['_xsrf'] = 'dummy'
datagen, headers = multipart_encode(post_data)
databuffer = b''.join(datagen)
headers['Cookie'] = "_xsrf=dummy"
response = self.fetch('/uploadproject',
method='POST',
headers=headers,
body=databuffer)
self.assertEqual(200, response.code)
with session_scope() as session:
project = session.query(Project).filter_by(
name=project_name).first()
self.assertIsNotNone(project)
self.assertEqual(project.name, project_name)
def test_simple(self):
p = encode.MultipartParam("foo", "bar")
boundary = "XYZXYZXYZ"
expected = unix2dos(b"""--XYZXYZXYZ
Content-Disposition: form-data; name="foo"
Content-Type: text/plain; charset=utf-8
bar
--XYZXYZXYZ--
""")
self.assertEqual(p.encode(boundary), expected[:-len(boundary) - 6])
self.assertEqual(p.get_size(boundary),
len(expected) - len(boundary) - 6)
self.assertEqual(encode.get_body_size([p], boundary), len(expected))
self.assertEqual(
encode.get_headers([p], boundary), {
'Content-Length': str(len(expected)),
'Content-Type': 'multipart/form-data; boundary=%s' % boundary
})
datagen, headers = encode.multipart_encode([p], boundary)
self.assertEqual(
headers, {
'Content-Length': str(len(expected)),
'Content-Type': 'multipart/form-data; boundary=%s' % boundary
})
self.assertEqual(b"".join(datagen), expected)
def test_post_create(self):
project_name = 'test_project'
postdata = {'project': project_name}
response = self.fetch('/delproject.json',
method='POST',
body=urlencode(postdata))
self.assertIn(response.code, [404, 200])
post_data = {}
post_data['egg'] = open(TEST_EGG_FILE, 'rb')
post_data['project'] = project_name
post_data['version'] = '1.0'
post_data['_xsrf'] = 'dummy'
datagen, headers = multipart_encode(post_data)
databuffer = b''.join([ensure_binary(x) for x in datagen])
headers['Cookie'] = "_xsrf=dummy"
response = self.fetch('/addversion.json',
method='POST',
headers=headers,
body=databuffer)
self.assertEqual(200, response.code)
with session_scope() as session:
project = session.query(Project)\
.filter_by(name=project_name).first()
self.assertIsNotNone(project)
self.assertEqual(project.name, project_name)
def test_multipart_encode_cb(self):
log = []
def cb(p, current, total):
log.append((p, current, total))
p = encode.MultipartParam("foo", "bar")
boundary = "XYZXYZXYZ"
datagen, headers = encode.multipart_encode([p], boundary, cb=cb)
b"".join(datagen)
l = int(headers['Content-Length'])
self.assertEquals(log[-1], (None, l, l))
def test_MultipartParam_cb(self):
log = []
def cb(p, current, total):
log.append((p, current, total))
p = encode.MultipartParam("foo", "bar", cb=cb)
boundary = "XYZXYZXYZ"
datagen, headers = encode.multipart_encode([p], boundary)
b"".join(datagen)
l = p.get_size(boundary)
self.assertEquals(log[-1], (p, l, l))
def test_stringio(self):
fp = StringIO("file data")
params = encode.MultipartParam.from_params([("foo", fp)])
boundary = "XYZXYZXYZ"
datagen, headers = encode.multipart_encode(params, boundary)
encoded = b"".join(datagen)
expected = unix2dos(b"""--XYZXYZXYZ
Content-Disposition: form-data; name="foo"
Content-Type: text/plain; charset=utf-8
file data
--XYZXYZXYZ--
""")
self.assertEqual(encoded, expected)
def _upload_test_project(self):
# upload a project
post_data = {}
post_data['egg'] = open(TEST_EGG_FILE, 'rb')
post_data['project'] = 'test_project'
post_data['version'] = '1.0'
datagen, headers = multipart_encode(post_data)
databuffer = ''.join(datagen)
response = self.fetch('/addversion.json',
method='POST',
headers=headers,
body=databuffer)
self.assertEqual(200, response.code)
def _upload_test_project(self):
post_data = {}
post_data['egg'] = open(
os.path.join(os.path.dirname(__file__),
'test_project-1.0-py2.7.egg'), 'rb')
post_data['project'] = 'test_project'
post_data['version'] = '1.0'
datagen, headers = multipart_encode(post_data)
self.populate_basic_authorization_header(headers)
databuffer = ''.join(datagen)
response = self.fetch('/addversion.json',
method='POST',
headers=headers,
body=databuffer)
self.assertEqual(200, response.code)
def test_reset_string(self):
p = encode.MultipartParam("foo", "bar")
boundary = "XYZXYZXYZ"
datagen, headers = encode.multipart_encode([p], boundary)
expected = unix2dos(b"""--XYZXYZXYZ
Content-Disposition: form-data; name="foo"
Content-Type: text/plain; charset=utf-8
bar
--XYZXYZXYZ--
""")
self.assertEquals(b"".join(datagen), expected)
datagen.reset()
self.assertEquals(b"".join(datagen), expected)
def test_post(self):
project_name = 'test_project'
post_data = {}
post_data['egg'] = open(TEST_EGG_FILE, 'rb')
post_data['project'] = project_name
post_data['version'] = '1.0'
post_data['_xsrf'] = 'dummy'
datagen, headers = multipart_encode(post_data)
databuffer = b''.join([ensure_binary(x) for x in datagen])
headers['Cookie'] = "_xsrf=dummy"
response = self.fetch('/addversion.json',
method='POST',
headers=headers,
body=databuffer)
self.assertEqual(400, response.code)
def test_job_complete(self):
project_name = 'test_project'
spider_name = 'success_spider'
node_id = self.register_node()
# schedule a job
with session_scope() as session:
session.query(SpiderExecutionQueue).delete()
session.commit()
run_spider_post_data = {'project': project_name, 'spider': spider_name}
res = self.fetch('/schedule.json', method='POST', body=urlencode(run_spider_post_data))
self.assertEqual(200, res.code)
# fetch a job
next_job_post_data = {'node_id': node_id}
headers = {'X-Dd-Nodeid': str(node_id)}
res = self.fetch('/executing/next_task', method='POST', body=urlencode(next_job_post_data),
headers=headers)
self.assertEqual(200, res.code)
task_id = json.loads(res.body)['data']['task']['task_id']
# job start
post_data = {'pid' : '1'}
headers = {'X-Dd-Nodeid': str(node_id)}
res = self.fetch('/jobs/%s/start' % task_id, method='POST', headers=headers, body=urlencode(post_data))
self.assertEqual(200, res.code)
# complete this job
post_data = {'task_id': task_id,
'status': 'success'}
post_data['log'] = BytesIO(b'some logs')
post_data['items'] = BytesIO(b'{"a" : "some items"}')
datagen, headers = multipart_encode(post_data)
headers['X-Dd-Nodeid'] = str(node_id)
#
res = self.fetch('/executing/complete', method='POST', headers=headers,
body_producer=MultipartRequestBodyProducer(datagen))
self.assertEqual(200, res.code)
with session_scope() as session:
complete_job = session.query(HistoricalJob).filter_by(id=task_id).first()
self.assertIsNotNone(complete_job)
self.assertEqual(2, complete_job.status)
def test_multiple_keys(self):
params = encode.MultipartParam.from_params([("key", "value1"),
("key", "value2")])
boundary = "XYZXYZXYZ"
datagen, headers = encode.multipart_encode(params, boundary)
encoded = b"".join(datagen)
expected = unix2dos(b"""--XYZXYZXYZ
Content-Disposition: form-data; name="key"
Content-Type: text/plain; charset=utf-8
value1
--XYZXYZXYZ
Content-Disposition: form-data; name="key"
Content-Type: text/plain; charset=utf-8
value2
--XYZXYZXYZ--
""")
self.assertEqual(encoded, expected)