def test_make_multipart(self):
field = RequestField('somename', 'data')
field.make_multipart(content_type='image/jpg',
content_location='/test')
assert (
field.render_headers() ==
'Content-Disposition: form-data; name="somename"\r\n'
'Content-Type: image/jpg\r\n'
'Content-Location: /test\r\n'
'\r\n')
def _add_multipart(parts):
mime_multipart_parts = list()
for name, (filename, data, content_type) in parts.items():
multipart_part = RequestField(name=name,
data=data,
filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
request, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
return request, content_type
def test_request_fields(self):
fields = [RequestField('k', b'v', filename='somefile.txt', headers={'Content-Type': 'image/jpeg'})]
encoded, content_type = encode_multipart_formdata(fields, boundary=BOUNDARY)
self.assertEquals(encoded,
b'--' + b(BOUNDARY) + b'\r\n'
b'Content-Type: image/jpeg\r\n'
b'\r\n'
b'v\r\n'
b'--' + b(BOUNDARY) + b'--\r\n'
)
def process_upload_items(items):
result = []
for key, val in items:
if isinstance(val, UploadContent):
headers = {'Content-Type': val.content_type}
field = RequestField(name=key,
data=val.content,
filename=val.filename,
headers=headers)
field.make_multipart(content_type=val.content_type)
result.append(field)
elif isinstance(val, UploadFile):
data = open(val.path, 'rb').read()
headers = {'Content-Type': val.content_type}
field = RequestField(name=key,
data=data,
filename=val.filename,
headers=headers)
field.make_multipart(content_type=val.content_type)
result.append(field)
else:
result.append((key, val))
return result
def upload_file(self, from_path, to_url):
url = construct_url_path(to_url)
self._display(HTTPMethod.POST, 'upload', url)
with open(from_path, 'rb') as src_file:
rf = RequestField('fileToUpload', src_file.read(), os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers = dict(BASE_HEADERS)
headers['Content-Type'] = content_type
headers['Content-Length'] = len(body)
dummy, response_data = self.connection.send(url, data=body, method=HTTPMethod.POST, headers=headers)
value = self._get_response_value(response_data)
self._display(HTTPMethod.POST, 'upload:response', value)
return self._response_to_json(value)
def upload_file(self, from_path, to_url):
url = construct_url_path(to_url)
with open(from_path, 'rb') as src_file:
rf = RequestField('fileToUpload', src_file.read(),
os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers = self._authorized_headers()
headers['Content-Type'] = content_type
headers['Content-Length'] = len(body)
dummy, response_data = self.connection.send(url,
data=body,
method=HTTPMethod.POST,
headers=headers)
return self._response_to_json(response_data.getvalue())
def __init__(self, check_only, chunksize=8192):
self.check_only = check_only
self.chunksize = chunksize
self.start = time.time()
self.uploaded = False
fields = [
('name', package),
('{}sum'.format(hashtype), hsh),
]
if check_only:
fields.append(('filename', filename))
else:
with open(filename, 'rb') as f:
rf = RequestField('file', f.read(), filename)
rf.make_multipart()
fields.append(rf)
self.data, content_type = encode_multipart_formdata(fields)
self.headers = {'Content-Type': content_type}
def parameters_to_multipart(self, params, collection_types):
"""Get parameters as list of tuples, formatting as json if value is collection_types
:param params: Parameters as list of two-tuples
:param dict collection_types: Parameter collection types
:return: Parameters as list of tuple or urllib3.fields.RequestField
"""
new_params = []
if collection_types is None:
collection_types = (dict)
for k, v in params.items() if isinstance(params, dict) else params: # noqa: E501
if isinstance(v, collection_types): # v is instance of collection_type, formatting as application/json
v = json.dumps(v, ensure_ascii=False).encode("utf-8")
field = RequestField(k, v)
field.make_multipart(content_type="application/json; charset=utf-8")
new_params.append(field)
else:
new_params.append((k, v))
return new_params
def test_request_fields(self) -> None:
fields = [
RequestField(
"k",
b"v",
filename="somefile.txt",
headers={"Content-Type": "image/jpeg"},
)
]
encoded, content_type = encode_multipart_formdata(fields,
boundary=BOUNDARY)
expected = (b"--" + BOUNDARY_BYTES + b"\r\n"
b"Content-Type: image/jpeg\r\n"
b"\r\n"
b"v\r\n"
b"--" + BOUNDARY_BYTES + b"--\r\n")
assert encoded == expected
def upload_file(self, from_path, to_url):
url = construct_url_path(to_url)
with open(from_path, 'rb') as src_file:
rf = RequestField('fileToUpload', src_file.read(),
os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers = self._authorized_headers()
headers['Content-Type'] = content_type
headers['Content-Length'] = len(body)
response, response_data = self.connection.send(url,
data=body,
method='POST',
headers=headers)
try:
ret = json.loads(to_text(response_data.getvalue()))
except:
raise ConnectionError(
'Response was not valid JSON, got {0}'.format(
response_data.getvalue()))
return ret
def test_render_part_html5_unicode_with_control_character(self):
field = RequestField("somename", "data")
param = field._render_part("filename", u("hello\x1A\x1B\x1C"))
assert param == u('filename="hello%1A\x1B%1C"')
def test_render_part_html5_ascii(self):
field = RequestField("somename", "data")
param = field._render_part("filename", b"name")
assert param == 'filename="name"'
def test_render_part_html5_unicode_escape(self):
field = RequestField("somename", "data")
param = field._render_part("filename", u("hello\\world\u0022"))
assert param == u('filename="hello\\\\world%22"')
def test_render_part_rfc2231_ascii(self):
field = RequestField("somename",
"data",
header_formatter=format_header_param_rfc2231)
param = field._render_part("filename", b"name")
assert param == 'filename="name"'
def test_render_part_html5_unicode(self):
field = RequestField("somename", "data")
param = field._render_part("filename", u("n\u00e4me"))
assert param == u('filename="n\u00e4me"')
def test_render_part_rfc2231_unicode(self):
field = RequestField('somename', 'data', header_formatter=format_header_param_rfc2231)
param = field._render_part('filename', u('n\u00e4me'))
assert param == "filename*=utf-8''n%C3%A4me"
def test_render_part_rfc2231_unicode(self):
field = RequestField("somename",
"data",
header_formatter=format_header_param_rfc2231)
param = field._render_part("filename", u("n\u00e4me"))
assert param == "filename*=utf-8''n%C3%A4me"
def test_render_part_html5_unicode(self):
field = RequestField('somename', 'data')
param = field._render_part('filename', u('n\u00e4me'))
assert param == u('filename="n\u00e4me"')
def test_render_part_html5_ascii(self):
field = RequestField('somename', 'data')
param = field._render_part('filename', b'name')
assert param == 'filename="name"'
def test_render_part_html5_unicode_escape(self):
field = RequestField('somename', 'data')
param = field._render_part('filename', u('hello\\world\u0022'))
assert param == u('filename="hello\\\\world%22"')
def test_render_part_html5_unicode_with_control_character(self):
field = RequestField('somename', 'data')
param = field._render_part('filename', u('hello\x1A\x1B\x1C'))
assert param == u('filename="hello%1A\x1B%1C"')
def upload_mub(module,
mgr_url,
mgr_username,
mgr_password,
validate_certs,
request_data,
headers,
ip_address,
timeout=10800):
endpoint = '/upgrade/bundles'
mub_type = 'url'
#headers = {}
if module.params['file'] is not None:
mub_type = 'file'
endpoint = endpoint + '?action=upload'
if mub_type == 'file':
file_path = module.params['file']
try:
file_data = open(file_path, 'rb')
atexit.register(file_data.close)
except Exception as e:
module.fail_json(msg='failed to open mub file %s Error: %s' %
(file_path, to_native(e)))
if os.stat(file_path).st_size == 0:
request_data = ''
else:
request_data = mmap.mmap(file_data.fileno(),
0,
access=mmap.ACCESS_READ)
atexit.register(request_data.close)
from urllib3 import encode_multipart_formdata
from urllib3.fields import RequestField
with open(file_path, 'rb') as src_file:
rf = RequestField('file', src_file.read(),
os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers['Content-Type'] = content_type
headers['Content-length'] = len(body)
if mub_type == 'url':
body = request_data
try:
(rc, resp) = request(mgr_url + endpoint,
data=body,
headers=headers,
method='POST',
url_username=mgr_username,
url_password=mgr_password,
validate_certs=validate_certs,
ignore_errors=True)
if rc == 200:
bundle_id = 'latest' #resp['bundle_id']
headers = dict(Accept="application/json")
headers['Content-Type'] = 'application/json'
try:
wait_for_operation_to_execute(
mgr_url, '/upgrade/bundles/%s/upload-status' % bundle_id,
mgr_username, mgr_password, validate_certs, ['status'],
['SUCCESS'], ['FAILED'])
except Exception as err:
module.fail_json(
msg='Error while uploading upgrade bundle. Error [%s]' %
to_native(err))
module.exit_json(
changed=True,
ip_address=ip_address,
response=resp,
message='The upgrade bundle %s got uploaded successfully.' %
module.params[mub_type])
else:
module.fail_json(msg='Failed to run upload mub. response code: {}'
' response: {}'.format(rc, resp))
except Exception as err:
module.fail_json(changed=True, msg="Error: {}".format(err))
def _encode_files(self, files, data):
"""
Method taken from models in requests library , usage is the same. Only difference is that you can add custom
boundary in 5-tuple version.
Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be
2-tuples (filename, fileobj),
3-tuples (filename, fileobj, contentype),
4-tuples (filename, fileobj, contentype, custom_headers) or
5-tuples (filename, fileobj, contentype, custom_headers, custom boundary).
example:
files = {'file': ('report.xls', body, 'application/vnd.ms-excel', {'Expires': '0'}, 'custom_boundary')}
"""
if not files:
raise ValueError("Files must be provided.")
elif isinstance(data, basestring):
raise ValueError("Data must not be a string.")
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, val in fields:
if isinstance(val, basestring) or not hasattr(val, "__iter__"):
val = [val]
for v in val:
if v is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append((
field.decode("utf-8")
if isinstance(field, bytes) else field,
v.encode("utf-8") if isinstance(v, str) else v,
))
for (k, v) in files:
# support for explicit filename
ft = None
fh = None
boundary = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
elif len(v) == 4:
fn, fp, ft, fh = v
else:
fn, fp, ft, fh, boundary = v
else:
fn = self.guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
elif hasattr(fp, "read"):
fdata = fp.read()
elif fp is None:
continue
else:
fdata = fp
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields, boundary)
return body, content_type
def _encode_files(files: dict, data: dict):
"""Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
or 4-tuples (filename, fileobj, contentype, custom_headers).
"""
if not files:
raise ValueError("Files must be provided.")
elif isinstance(data, str):
raise ValueError("Data must not be a string.")
index = {}
index.update(data)
index.update(files)
index = Utils.reorder_by_hash_code(index)
new_fields = []
for key, value in index.items():
if key in files:
# support for explicit filename
ft = None
fh = None
if isinstance(value, (tuple, list)):
if len(value) == 2:
fn, fp = value
elif len(value) == 3:
fn, fp, ft = value
else:
fn, fp, ft, fh = value
else:
fn = guess_filename(value) or key
fp = value
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
elif hasattr(fp, 'read'):
fdata = fp.read()
elif fp is None:
continue
else:
fdata = fp
rf = RequestField(name=key,
data=fdata,
filename=fn,
headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
elif key in data:
if isinstance(value, str) or not hasattr(value, '__iter__'):
value = [value]
for value in value:
if value is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(value, bytes):
value = str(value)
new_fields.append((key.decode('utf-8') if isinstance(
key, bytes) else key, value.encode('utf-8') if
isinstance(value, str) else value))
body, content_type = encode_multipart_formdata(new_fields)
# TODO: compress body
return body, content_type
def test_render_part(self):
field = RequestField('somename', 'data')
param = field._render_part('filename', u('n\u00e4me'))
assert param == "filename*=utf-8''n%C3%A4me"
def test_render_part_rfc2231_ascii(self):
field = RequestField('somename', 'data', header_formatter=format_header_param_rfc2231)
param = field._render_part('filename', b'name')
assert param == 'filename="name"'
def _encode_files(files, data):
"""Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
or 4-tuples (filename, fileobj, contentype, custom_headers).
"""
if (not files):
raise ValueError("Files must be provided.")
elif isinstance(data, basestring):
raise ValueError("data must not be a string.")
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, val in fields:
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
for v in val:
if v is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append(
(field.decode('utf-8') if isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str) else v))
for (k, v) in files:
# support for explicit filename
ft = None
fh = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
else:
fn, fp, ft, fh = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
elif hasattr(fp, 'read'):
fdata = fp.read()
elif fp is None:
continue
else:
fdata = fp
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields)
return body, content_type
def test_render_unicode_bytes_py2(self):
field = RequestField('somename', 'data')
param = field._render_part('filename', 'n\xc3\xa4me')
assert param == "filename*=utf-8''n%C3%A4me"
