def test_make_multipart_empty_filename(self):
field = RequestField("somename", "data", "")
field.make_multipart(content_type="application/octet-stream")
assert (field.render_headers(
) == 'Content-Disposition: form-data; name="somename"; filename=""\r\n'
"Content-Type: application/octet-stream\r\n"
"\r\n")
def test_make_multipart_empty_filename(self):
field = RequestField('somename', 'data', '')
field.make_multipart(content_type='application/octet-stream')
assert (field.render_headers(
) == 'Content-Disposition: form-data; name="somename"; filename=""\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n')
def _encode_files(files, data):
"""Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
or 4-tuples (filename, fileobj, contentype, custom_headers).
"""
if (not files):
raise ValueError("Files must be provided.")
elif isinstance(data, basestring):
raise ValueError("Data must not be a string.")
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, val in fields:
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
for v in val:
if v is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append(
(field.decode('utf-8')
if isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str) else v))
for (k, v) in files:
# support for explicit filename
ft = None
fh = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
else:
fn, fp, ft, fh = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
else:
fdata = fp.read()
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields)
return body, content_type
def test_make_multipart_empty_filename(self):
field = RequestField('somename', 'data', '')
field.make_multipart(content_type='application/octet-stream')
assert (
field.render_headers() ==
'Content-Disposition: form-data; name="somename"; filename=""\r\n'
'Content-Type: application/octet-stream\r\n'
'\r\n')
def test_make_multipart(self):
field = RequestField('somename', 'data')
field.make_multipart(content_type='image/jpg', content_location='/test')
self.assertEqual(field.render_headers(),
'Content-Disposition: form-data; name="somename"\r\n'
'Content-Type: image/jpg\r\n'
'Content-Location: /test\r\n'
'\r\n')
def _encode_files(files, data):
"""Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
2-tuples. Order is retained if data is a list of 2-tuples but arbitrary
if parameters are supplied as a dict.
"""
if (not files):
raise ValueError("Files must be provided.")
elif isinstance(data, basestring):
raise ValueError("Data must not be a string.")
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, val in fields:
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
for v in val:
if v is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append(
(field.decode('utf-8') if isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str) else v))
for (k, v) in files:
# support for explicit filename
ft = None
fh = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
else:
fn, fp, ft, fh = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
else:
fdata = fp.read()
rf = RequestField(name=k, data=fdata,
filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields)
return body, content_type
def test_make_multipart(self):
field = RequestField('somename', 'data')
field.make_multipart(content_type='image/jpg',
content_location='/test')
assert (field.render_headers() ==
'Content-Disposition: form-data; name="somename"\r\n'
'Content-Type: image/jpg\r\n'
'Content-Location: /test\r\n'
'\r\n')
def _add_multipart(parts):
mime_multipart_parts = list()
for name, (filename, data, content_type) in parts.items():
multipart_part = RequestField(name=name, data=data, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
request, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return request, content_type
def test_make_multipart(self):
field = RequestField("somename", "data")
field.make_multipart(content_type="image/jpg",
content_location="/test")
assert (field.render_headers() ==
'Content-Disposition: form-data; name="somename"\r\n'
"Content-Type: image/jpg\r\n"
"Content-Location: /test\r\n"
"\r\n")
def test_make_multipart(self):
field = RequestField("somename", "data")
field.make_multipart(content_type="image/jpg", content_location="/test")
self.assertEqual(
field.render_headers(),
'Content-Disposition: form-data; name="somename"\r\n'
"Content-Type: image/jpg\r\n"
"Content-Location: /test\r\n"
"\r\n",
)
def _encode_form_data(payload_file):
"""Encode multipart/form-data for file upload."""
fields = []
f_name, f_data, f_type = payload_file.get("file")
f_binary = f_data.read()
req_field = RequestField(name="file", data=f_binary, filename=f_name)
req_field.make_multipart(content_type=f_type)
fields.append(req_field)
data, content_type = encode_multipart_formdata(fields)
return data, content_type
def encode_params(self, data=None, files=None, **kwargs):
"""
Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be string (filepath), 2-tuples (filename, fileobj), 3-tuples
(filename, fileobj, contentype) or 4-tuples (filename, fileobj, contentype, custom_headers).
"""
if isinstance(data, basestring):
raise ValueError("Data must not be a string.")
# optional args
boundary = kwargs.get("boundary", None)
output_str = kwargs.get("output_str", self.output_str)
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, value in fields:
ctype = None
if isinstance(value, (tuple, list)) and len(value) == 2:
val, ctype = value
else:
val = value
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
for v in val:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = to_string(v, lang=output_str)
field = field.decode('utf-8') if isinstance(field,
bytes) else field
v = v.encode('utf-8') if isinstance(v, str) else v
rf = RequestField(name=field, data=v)
rf.make_multipart(content_type=ctype)
new_fields.append(rf)
for (k, v) in files:
fn, fdata, ft, fh = guess_file_name_stream_type_header(v)
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
if boundary is None:
boundary = self.boundary
body, content_type = encode_multipart_formdata(new_fields,
boundary=boundary)
return body, content_type
def upload_file(self, from_path, to_url):
url = construct_url_path(to_url)
self._display(HTTPMethod.POST, 'upload', url)
with open(from_path, 'rb') as src_file:
rf = RequestField('fileToUpload', src_file.read(), os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers = dict(BASE_HEADERS)
headers['Content-Type'] = content_type
headers['Content-Length'] = len(body)
dummy, response_data = self.connection.send(url, data=body, method=HTTPMethod.POST, headers=headers)
value = self._get_response_value(response_data)
self._display(HTTPMethod.POST, 'upload:response', value)
return self._response_to_json(value)
def upload_file(self, from_path, to_url):
url = construct_url_path(to_url)
with open(from_path, 'rb') as src_file:
rf = RequestField('fileToUpload', src_file.read(),
os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers = self._authorized_headers()
headers['Content-Type'] = content_type
headers['Content-Length'] = len(body)
dummy, response_data = self.connection.send(url,
data=body,
method=HTTPMethod.POST,
headers=headers)
return self._response_to_json(response_data.getvalue())
def process_upload_items(items):
result = []
for key, val in items:
if isinstance(val, UploadContent):
headers = {"Content-Type": val.content_type}
field = RequestField(name=key, data=val.content, filename=val.filename, headers=headers)
field.make_multipart(content_type=val.content_type)
result.append(field)
elif isinstance(val, UploadFile):
data = open(val.path, "rb").read()
headers = {"Content-Type": val.content_type}
field = RequestField(name=key, data=data, filename=val.filename, headers=headers)
field.make_multipart(content_type=val.content_type)
result.append(field)
else:
result.append((key, val))
return result
def __init__(self, check_only, chunksize=8192):
self.check_only = check_only
self.chunksize = chunksize
self.start = time.time()
self.uploaded = False
fields = [
('name', package),
('{}sum'.format(hashtype), hsh),
]
if check_only:
fields.append(('filename', filename))
else:
with open(filename, 'rb') as f:
rf = RequestField('file', f.read(), filename)
rf.make_multipart()
fields.append(rf)
self.data, content_type = encode_multipart_formdata(fields)
self.headers = {'Content-Type': content_type}
def parameters_to_multipart(self, params, collection_types):
"""Get parameters as list of tuples, formatting as json if value is collection_types
:param params: Parameters as list of two-tuples
:param dict collection_types: Parameter collection types
:return: Parameters as list of tuple or urllib3.fields.RequestField
"""
new_params = []
if collection_types is None:
collection_types = (dict)
for k, v in params.items() if isinstance(params, dict) else params: # noqa: E501
if isinstance(v, collection_types): # v is instance of collection_type, formatting as application/json
v = json.dumps(v, ensure_ascii=False).encode("utf-8")
field = RequestField(k, v)
field.make_multipart(content_type="application/json; charset=utf-8")
new_params.append(field)
else:
new_params.append((k, v))
return new_params
def encode_body(self):
new_fields = []
for field, val in self.form_fields:
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
for v in val:
if v is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append(
(field.decode('utf-8') if isinstance(field, bytes) else field,
v.encode('utf-8') if isinstance(v, str) else v))
if not self.files:
self.files = {}
for k, v in self.files.items():
# support for explicit filename
ft = None
fh = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
else:
fn, fp, ft, fh = v
else:
fn = guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
else:
fdata = fp.read()
rf = RequestField(name=k, data=fdata,
filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields)
return body,content_type
def process_upload_items(items):
result = []
for key, val in items:
if isinstance(val, UploadContent):
headers = {'Content-Type': val.content_type}
field = RequestField(name=key, data=val.content,
filename=val.filename, headers=headers)
field.make_multipart(content_type=val.content_type)
result.append(field)
elif isinstance(val, UploadFile):
data = open(val.path, 'rb').read()
headers = {'Content-Type': val.content_type}
field = RequestField(name=key, data=data,
filename=val.filename, headers=headers)
field.make_multipart(content_type=val.content_type)
result.append(field)
else:
result.append((key, val))
return result
def upload_file(self, from_path, to_url):
url = construct_url_path(to_url)
with open(from_path, 'rb') as src_file:
rf = RequestField('fileToUpload', src_file.read(),
os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers = self._authorized_headers()
headers['Content-Type'] = content_type
headers['Content-Length'] = len(body)
response, response_data = self.connection.send(url,
data=body,
method='POST',
headers=headers)
try:
ret = json.loads(to_text(response_data.getvalue()))
except:
raise ConnectionError(
'Response was not valid JSON, got {0}'.format(
response_data.getvalue()))
return ret
def _encode_files(files: dict, data: dict):
"""Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
or 4-tuples (filename, fileobj, contentype, custom_headers).
"""
if not files:
raise ValueError("Files must be provided.")
elif isinstance(data, str):
raise ValueError("Data must not be a string.")
index = {}
index.update(data)
index.update(files)
index = Utils.reorder_by_hash_code(index)
new_fields = []
for key, value in index.items():
if key in files:
# support for explicit filename
ft = None
fh = None
if isinstance(value, (tuple, list)):
if len(value) == 2:
fn, fp = value
elif len(value) == 3:
fn, fp, ft = value
else:
fn, fp, ft, fh = value
else:
fn = guess_filename(value) or key
fp = value
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
elif hasattr(fp, 'read'):
fdata = fp.read()
elif fp is None:
continue
else:
fdata = fp
rf = RequestField(name=key,
data=fdata,
filename=fn,
headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
elif key in data:
if isinstance(value, str) or not hasattr(value, '__iter__'):
value = [value]
for value in value:
if value is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(value, bytes):
value = str(value)
new_fields.append((key.decode('utf-8') if isinstance(
key, bytes) else key, value.encode('utf-8') if
isinstance(value, str) else value))
body, content_type = encode_multipart_formdata(new_fields)
# TODO: compress body
return body, content_type
def _encode_files(self, files, data):
"""
Method taken from models in requests library , usage is the same. Only difference is that you can add custom
boundary in 5-tuple version.
Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be
2-tuples (filename, fileobj),
3-tuples (filename, fileobj, contentype),
4-tuples (filename, fileobj, contentype, custom_headers) or
5-tuples (filename, fileobj, contentype, custom_headers, custom boundary).
example:
files = {'file': ('report.xls', body, 'application/vnd.ms-excel', {'Expires': '0'}, 'custom_boundary')}
"""
if not files:
raise ValueError("Files must be provided.")
elif isinstance(data, basestring):
raise ValueError("Data must not be a string.")
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, val in fields:
if isinstance(val, basestring) or not hasattr(val, "__iter__"):
val = [val]
for v in val:
if v is not None:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = str(v)
new_fields.append((
field.decode("utf-8")
if isinstance(field, bytes) else field,
v.encode("utf-8") if isinstance(v, str) else v,
))
for (k, v) in files:
# support for explicit filename
ft = None
fh = None
boundary = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
elif len(v) == 4:
fn, fp, ft, fh = v
else:
fn, fp, ft, fh, boundary = v
else:
fn = self.guess_filename(v) or k
fp = v
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
elif hasattr(fp, "read"):
fdata = fp.read()
elif fp is None:
continue
else:
fdata = fp
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
body, content_type = encode_multipart_formdata(new_fields, boundary)
return body, content_type
def encode_params(self, data=None, files=None, **kwargs):
"""
Build the body for a multipart/form-data request.
Will successfully encode files when passed as a dict or a list of
tuples. Order is retained if data is a list of tuples but arbitrary
if parameters are supplied as a dict.
The tuples may be string (filepath), 2-tuples (filename, fileobj), 3-tuples (filename, fileobj, contentype)
or 4-tuples (filename, fileobj, contentype, custom_headers).
"""
if isinstance(data, basestring):
raise ValueError("Data must not be a string.")
# optional args
boundary = kwargs.get("boundary", None)
output_str = kwargs.get("output_str", self.output_str)
new_fields = []
fields = to_key_val_list(data or {})
files = to_key_val_list(files or {})
for field, value in fields:
ctype = None
if isinstance(value, (tuple, list)) and len(value) == 2:
val, ctype = value
else:
val = value
if isinstance(val, basestring) or not hasattr(val, '__iter__'):
val = [val]
for v in val:
# Don't call str() on bytestrings: in Py3 it all goes wrong.
if not isinstance(v, bytes):
v = to_string(v, lang=output_str)
field = field.decode('utf-8') if isinstance(field, bytes) else field
v = v.encode('utf-8') if isinstance(v, str) else v
rf = RequestField(name=field, data=v)
rf.make_multipart(content_type=ctype)
new_fields.append(rf)
for (k, v) in files:
# support for explicit filename
ft = None
fh = None
if isinstance(v, (tuple, list)):
if len(v) == 2:
fn, fp = v
elif len(v) == 3:
fn, fp, ft = v
else:
fn, fp, ft, fh = v
else:
fn, fp = guess_filename_stream(v)
ft = guess_content_type(fn)
if isinstance(fp, (str, bytes, bytearray)):
fdata = fp
else:
fdata = fp.read()
rf = RequestField(name=k, data=fdata, filename=fn, headers=fh)
rf.make_multipart(content_type=ft)
new_fields.append(rf)
if boundary is None:
boundary = self.boundary
body, content_type = encode_multipart_formdata(new_fields, boundary=boundary)
return body, content_type
def upload_mub(module,
mgr_url,
mgr_username,
mgr_password,
validate_certs,
request_data,
headers,
ip_address,
timeout=10800):
endpoint = '/upgrade/bundles'
mub_type = 'url'
#headers = {}
if module.params['file'] is not None:
mub_type = 'file'
endpoint = endpoint + '?action=upload'
if mub_type == 'file':
file_path = module.params['file']
try:
file_data = open(file_path, 'rb')
atexit.register(file_data.close)
except Exception as e:
module.fail_json(msg='failed to open mub file %s Error: %s' %
(file_path, to_native(e)))
if os.stat(file_path).st_size == 0:
request_data = ''
else:
request_data = mmap.mmap(file_data.fileno(),
0,
access=mmap.ACCESS_READ)
atexit.register(request_data.close)
from urllib3 import encode_multipart_formdata
from urllib3.fields import RequestField
with open(file_path, 'rb') as src_file:
rf = RequestField('file', src_file.read(),
os.path.basename(src_file.name))
rf.make_multipart()
body, content_type = encode_multipart_formdata([rf])
headers['Content-Type'] = content_type
headers['Content-length'] = len(body)
if mub_type == 'url':
body = request_data
try:
(rc, resp) = request(mgr_url + endpoint,
data=body,
headers=headers,
method='POST',
url_username=mgr_username,
url_password=mgr_password,
validate_certs=validate_certs,
ignore_errors=True)
if rc == 200:
bundle_id = 'latest' #resp['bundle_id']
headers = dict(Accept="application/json")
headers['Content-Type'] = 'application/json'
try:
wait_for_operation_to_execute(
mgr_url, '/upgrade/bundles/%s/upload-status' % bundle_id,
mgr_username, mgr_password, validate_certs, ['status'],
['SUCCESS'], ['FAILED'])
except Exception as err:
module.fail_json(
msg='Error while uploading upgrade bundle. Error [%s]' %
to_native(err))
module.exit_json(
changed=True,
ip_address=ip_address,
response=resp,
message='The upgrade bundle %s got uploaded successfully.' %
module.params[mub_type])
else:
module.fail_json(msg='Failed to run upload mub. response code: {}'
' response: {}'.format(rc, resp))
except Exception as err:
module.fail_json(changed=True, msg="Error: {}".format(err))
