def test_fieldstorage_multipart_leading_whitespace(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "---------------------------721837373350705526688164684"
POSTDATA = """-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="id"
1234
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="title"
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="file"; filename="test.txt"
Content-Type: text/plain
Testing 123.
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="submit"
Add\x20
-----------------------------721837373350705526688164684--
"""
env = {
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE':
'multipart/form-data; boundary={}'.format(BOUNDARY),
'CONTENT_LENGTH': '560'
}
# Add some leading whitespace to our post data that will cause the
# first line to not be the innerboundary.
fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1'))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 4
expect = [{
'name': 'id',
'filename': None,
'value': '1234'
}, {
'name': 'title',
'filename': None,
'value': ''
}, {
'name': 'file',
'filename': 'test.txt',
'value': b'Testing 123.\n'
}, {
'name': 'submit',
'filename': None,
'value': ' Add '
}]
for x in range(len(fs.list)):
for k, exp in expect[x].items():
got = getattr(fs.list[x], k)
assert got == exp
def test_my_fieldstorage_part_content_length(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "4ddfd368-cb07-4b9e-b003-876010298a6c"
POSTDATA = """--4ddfd368-cb07-4b9e-b003-876010298a6c
Content-Disposition: form-data; name="object"; filename="file.txt"
Content-Type: text/plain
Content-Length: 5
Content-Transfer-Encoding: 7bit
ADMIN
--4ddfd368-cb07-4b9e-b003-876010298a6c
Content-Disposition: form-data; name="sign_date"
Content-Type: application/json; charset=UTF-8
Content-Length: 22
Content-Transfer-Encoding: 7bit
"2016-11-23T12:22:41Z"
--4ddfd368-cb07-4b9e-b003-876010298a6c
Content-Disposition: form-data; name="staffId"
Content-Type: text/plain; charset=UTF-8
Content-Length: 5
Content-Transfer-Encoding: 7bit
ADMIN
--4ddfd368-cb07-4b9e-b003-876010298a6c--"""
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE":
"multipart/form-data; boundary={}".format(BOUNDARY),
"CONTENT_LENGTH": str(len(POSTDATA)),
}
fp = BytesIO(POSTDATA.encode("latin-1"))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 3
expect = [
{
"name": "object",
"filename": "file.txt",
"value": b"ADMIN"
},
{
"name": "sign_date",
"filename": None,
"value": '"2016-11-23T12:22:41Z"'
},
{
"name": "staffId",
"filename": None,
"value": "ADMIN"
},
]
for x in range(len(fs.list)):
for k, exp in expect[x].items():
got = getattr(fs.list[x], k)
assert got == exp
def test_fieldstorage_not_multipart(self):
from webob.compat import cgi_FieldStorage
POSTDATA = b'{"name": "Bert"}'
env = {
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'text/plain',
'CONTENT_LENGTH': str(len(POSTDATA))
}
fp = BytesIO(POSTDATA)
fs = cgi_FieldStorage(fp, environ=env)
assert fs.list is None
assert fs.value == b'{"name": "Bert"}'
def test_fieldstorage_not_multipart(self):
from webob.compat import cgi_FieldStorage
POSTDATA = b'{"name": "Bert"}'
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": "text/plain",
"CONTENT_LENGTH": str(len(POSTDATA)),
}
fp = BytesIO(POSTDATA)
fs = cgi_FieldStorage(fp, environ=env)
assert fs.list is None
assert fs.value == b'{"name": "Bert"}'
def test_fieldstorage_multipart_leading_whitespace(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "---------------------------721837373350705526688164684"
POSTDATA = """-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="id"
1234
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="title"
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="file"; filename="test.txt"
Content-Type: text/plain
Testing 123.
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="submit"
Add\x20
-----------------------------721837373350705526688164684--
"""
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": "multipart/form-data; boundary={}".format(BOUNDARY),
"CONTENT_LENGTH": "560",
}
# Add some leading whitespace to our post data that will cause the
# first line to not be the innerboundary.
fp = BytesIO(b"\r\n" + POSTDATA.encode("latin-1"))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 4
expect = [
{"name": "id", "filename": None, "value": "1234"},
{"name": "title", "filename": None, "value": ""},
{"name": "file", "filename": "test.txt", "value": b"Testing 123.\n"},
{"name": "submit", "filename": None, "value": " Add "},
]
for x in range(len(fs.list)):
for k, exp in expect[x].items():
got = getattr(fs.list[x], k)
assert got == exp
def test_fieldstorage_part_content_length(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "JfISa01"
POSTDATA = """--JfISa01
Content-Disposition: form-data; name="submit-name"
Content-Length: 5
Larry
--JfISa01"""
env = {
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
'CONTENT_LENGTH': str(len(POSTDATA))}
fp = BytesIO(POSTDATA.encode('latin-1'))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 1
assert fs.list[0].name == 'submit-name'
assert fs.list[0].value == 'Larry'
def test_my_fieldstorage_part_content_length(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "4ddfd368-cb07-4b9e-b003-876010298a6c"
POSTDATA = """--4ddfd368-cb07-4b9e-b003-876010298a6c
Content-Disposition: form-data; name="object"; filename="file.txt"
Content-Type: text/plain
Content-Length: 5
Content-Transfer-Encoding: 7bit
ADMIN
--4ddfd368-cb07-4b9e-b003-876010298a6c
Content-Disposition: form-data; name="sign_date"
Content-Type: application/json; charset=UTF-8
Content-Length: 22
Content-Transfer-Encoding: 7bit
"2016-11-23T12:22:41Z"
--4ddfd368-cb07-4b9e-b003-876010298a6c
Content-Disposition: form-data; name="staffId"
Content-Type: text/plain; charset=UTF-8
Content-Length: 5
Content-Transfer-Encoding: 7bit
ADMIN
--4ddfd368-cb07-4b9e-b003-876010298a6c--"""
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": "multipart/form-data; boundary={}".format(BOUNDARY),
"CONTENT_LENGTH": str(len(POSTDATA)),
}
fp = BytesIO(POSTDATA.encode("latin-1"))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 3
expect = [
{"name": "object", "filename": "file.txt", "value": b"ADMIN"},
{"name": "sign_date", "filename": None, "value": '"2016-11-23T12:22:41Z"'},
{"name": "staffId", "filename": None, "value": "ADMIN"},
]
for x in range(len(fs.list)):
for k, exp in expect[x].items():
got = getattr(fs.list[x], k)
assert got == exp
def test_fieldstorage_multipart_leading_whitespace(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "---------------------------721837373350705526688164684"
POSTDATA = """-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="id"
1234
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="title"
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="file"; filename="test.txt"
Content-Type: text/plain
Testing 123.
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="submit"
Add\x20
-----------------------------721837373350705526688164684--
"""
env = {
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE': 'multipart/form-data; boundary={}'.format(BOUNDARY),
'CONTENT_LENGTH': '560'}
# Add some leading whitespace to our post data that will cause the
# first line to not be the innerboundary.
fp = BytesIO(b"\r\n" + POSTDATA.encode('latin-1'))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 4
expect = [{'name':'id', 'filename':None, 'value':'1234'},
{'name':'title', 'filename':None, 'value':''},
{'name':'file', 'filename':'test.txt', 'value':b'Testing 123.\n'},
{'name':'submit', 'filename':None, 'value':' Add '}]
for x in range(len(fs.list)):
for k, exp in expect[x].items():
got = getattr(fs.list[x], k)
assert got == exp
def test_fieldstorage_part_content_length(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "JfISa01"
POSTDATA = """--JfISa01
Content-Disposition: form-data; name="submit-name"
Content-Length: 5
Larry
--JfISa01"""
env = {
'REQUEST_METHOD': 'POST',
'CONTENT_TYPE':
'multipart/form-data; boundary={}'.format(BOUNDARY),
'CONTENT_LENGTH': str(len(POSTDATA))
}
fp = BytesIO(POSTDATA.encode('latin-1'))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 1
assert fs.list[0].name == 'submit-name'
assert fs.list[0].value == 'Larry'
def test_fieldstorage_part_content_length(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "JfISa01"
POSTDATA = """--JfISa01
Content-Disposition: form-data; name="submit-name"
Content-Length: 5
Larry
--JfISa01"""
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE": "multipart/form-data; boundary={}".format(BOUNDARY),
"CONTENT_LENGTH": str(len(POSTDATA)),
}
fp = BytesIO(POSTDATA.encode("latin-1"))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 1
assert fs.list[0].name == "submit-name"
assert fs.list[0].value == "Larry"
def test_fieldstorage_part_content_length(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "JfISa01"
POSTDATA = """--JfISa01
Content-Disposition: form-data; name="submit-name"
Content-Length: 5
Larry
--JfISa01"""
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE":
"multipart/form-data; boundary={}".format(BOUNDARY),
"CONTENT_LENGTH": str(len(POSTDATA)),
}
fp = BytesIO(POSTDATA.encode("latin-1"))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 1
assert fs.list[0].name == "submit-name"
assert fs.list[0].value == "Larry"
def _upload_test_data(self):
"""
Upload test datasets, which are defined in the <test></test>
section of the provided tool.
"""
if not self._tool.tests:
raise ValueError('Tests are not defined.')
self._test = self._tool.tests[0]
# All inputs with the type 'data'
self._data_inputs = {
x.name: x
for x in self._tool.input_params if x.type == 'data'
}
# Datasets from the <test></test> section
test_datasets = {
input_name: self._test.inputs[input_name][0]
for input_name in self._data_inputs.keys()
if input_name in self._test.inputs.keys()
}
# Conditional datasets
for name, value in self._test.inputs.items():
if '|' in name:
input_name = name.split('|')[1]
if input_name in self._data_inputs.keys():
test_datasets.update(
{input_name: self._test.inputs[name][0]})
if not test_datasets.keys():
not_supported_input_types = [
k for k, v in self._tool.inputs.items()
if v.type == 'repeat' or v.type == 'data_collection'
]
if not_supported_input_types:
raise ValueError('Not supported input types.')
else:
# Some tests don't have data inputs at all,
# so we can generate a tour without them
self._use_datasets = False
return
# Upload all test datasets
for input_name, input in self._data_inputs.items():
if input_name in test_datasets.keys():
filename = test_datasets[input_name]
input_path = self._tool.test_data_path(filename)
if not input_path:
raise ValueError('Test dataset "%s" doesn\'t exist.' %
input_name)
upload_tool = self._trans.app.toolbox.get_tool('upload1')
filename = os.path.basename(input_path)
with open(input_path, 'rb') as f:
content = f.read()
headers = {
'content-disposition':
'form-data; name="{}"; filename="{}"'.format(
'files_0|file_data', filename),
}
input_file = cgi_FieldStorage(headers=headers)
input_file.file = input_file.make_file()
input_file.file.write(content)
inputs = {
'dbkey': '?', # is it always a question mark?
'file_type': input.extensions[0],
'files_0|type': 'upload_dataset',
'files_0|space_to_tab': None,
'files_0|to_posix_lines': 'Yes',
'files_0|file_data': input_file,
}
params = Params(inputs, sanitize=False)
incoming = params.__dict__
output = upload_tool.handle_input(self._trans,
incoming,
history=None)
job_errors = output.get('job_errors', [])
if job_errors:
# self._errors.extend(job_errors)
raise ValueError('Cannot upload a dataset.')
else:
self._hids.update(
{input_name: output['out_data'][0][1].hid})
def test_fieldstorage_multipart_leading_whitespace(self):
from webob.compat import cgi_FieldStorage
BOUNDARY = "---------------------------721837373350705526688164684"
POSTDATA = """-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="id"
1234
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="title"
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="file"; filename="test.txt"
Content-Type: text/plain
Testing 123.
-----------------------------721837373350705526688164684
Content-Disposition: form-data; name="submit"
Add\x20
-----------------------------721837373350705526688164684--
"""
env = {
"REQUEST_METHOD": "POST",
"CONTENT_TYPE":
"multipart/form-data; boundary={}".format(BOUNDARY),
"CONTENT_LENGTH": "560",
}
# Add some leading whitespace to our post data that will cause the
# first line to not be the innerboundary.
fp = BytesIO(b"\r\n" + POSTDATA.encode("latin-1"))
fs = cgi_FieldStorage(fp, environ=env)
assert len(fs.list) == 4
expect = [
{
"name": "id",
"filename": None,
"value": "1234"
},
{
"name": "title",
"filename": None,
"value": ""
},
{
"name": "file",
"filename": "test.txt",
"value": b"Testing 123.\n"
},
{
"name": "submit",
"filename": None,
"value": " Add "
},
]
for x in range(len(fs.list)):
for k, exp in expect[x].items():
got = getattr(fs.list[x], k)
assert got == exp
def _upload_test_data(self):
"""
Upload test datasets, which are defined in the <test></test>
section of the provided tool.
"""
if not self._tool.tests:
raise ValueError('Tests are not defined.')
self._test = self._tool.tests[0]
# All inputs with the type 'data'
self._data_inputs = {x.name: x for x in self._tool.input_params if
x.type == 'data'}
# Datasets from the <test></test> section
test_datasets = {
input_name: self._test.inputs[input_name][0]
for input_name in self._data_inputs.keys()
if input_name in self._test.inputs.keys()
}
# Conditional datasets
for name, value in self._test.inputs.items():
if '|' in name:
input_name = name.split('|')[1]
if input_name in self._data_inputs.keys():
test_datasets.update({
input_name: self._test.inputs[name][0]
})
if not test_datasets.keys():
not_supported_input_types = [
k for k, v in self._tool.inputs.items() if
v.type == 'repeat' or v.type == 'data_collection'
]
if not_supported_input_types:
raise ValueError('Not supported input types.')
else:
# Some tests don't have data inputs at all,
# so we can generate a tour without them
self._use_datasets = False
return
# Upload all test datasets
for input_name, input in self._data_inputs.items():
if input_name in test_datasets.keys():
filename = test_datasets[input_name]
input_path = self._tool.test_data_path(filename)
if not input_path:
raise ValueError('Test dataset "%s" doesn\'t exist.' % input_name)
upload_tool = self._trans.app.toolbox.get_tool('upload1')
filename = os.path.basename(input_path)
with open(input_path, 'rb') as f:
content = f.read()
headers = {
'content-disposition':
'form-data; name="{}"; filename="{}"'.format(
'files_0|file_data', filename
),
}
input_file = cgi_FieldStorage(headers=headers)
input_file.file = input_file.make_file()
input_file.file.write(content)
inputs = {
'dbkey': '?', # is it always a question mark?
'file_type': input.extensions[0],
'files_0|type': 'upload_dataset',
'files_0|space_to_tab': None,
'files_0|to_posix_lines': 'Yes',
'files_0|file_data': input_file,
}
params = Params(inputs, sanitize=False)
incoming = params.__dict__
output = upload_tool.handle_input(self._trans, incoming,
history=None)
job_errors = output.get('job_errors', [])
if job_errors:
# self._errors.extend(job_errors)
raise ValueError('Cannot upload a dataset.')
else:
self._hids.update({
input_name: output['out_data'][0][1].hid
})