def _iter_field(cls, field_name, field):
file_name = None
file_type = None
file_headers = None
if field and isinstance(field, (list, tuple)):
if all([isinstance(f, (list, tuple)) for f in field]):
for f in field:
yield next(cls._iter_field(field_name, f))
else:
raise StopIteration()
if len(field) == 2:
file_name, file_pointer = field
elif len(field) == 3:
file_name, file_pointer, file_type = field
else:
file_name, file_pointer, file_type, file_headers = field
else:
file_pointer = field
field = RequestField(name=field_name,
data=file_pointer,
filename=file_name,
headers=file_headers)
field.make_multipart(content_type=file_type)
yield field
def _do_upload(self, upload_request):
c_path = upload_request.path
base_name = c_path.base_name()
parent_c_path = c_path.parent()
cm_root = self._load_folders_structure()
cm_parent_folder = cm_root.get_folder(parent_c_path)
if not cm_parent_folder:
# parent folder of given path does exist => folders needs to be created
cm_parent_folder = self._create_intermediary_folders(cm_root, parent_c_path)
# Check: is it an existing folder ?
cm_folder = cm_parent_folder.get_child_by_name(base_name)
if cm_folder:
# The CPath corresponds to an existing folder, upload is not possible
raise CInvalidFileTypeError(c_path, True)
url = self._build_rest_url("documents") + cm_parent_folder.file_id
in_stream = upload_request.byte_source().open_stream()
rf_bin = RequestField(name="bin", data=in_stream, filename=c_path.base_name())
# Cloudme does not support UTF-8 encoded filenames that look like:
# filename*=UTF-8''...
# Instead we send raw UTF-8 bytes in between quotes (filename cannot contain any quote)
rf_bin.headers["Content-Disposition"] = 'form-data; name="bin"; filename="%s"' % base_name
rf_bin.headers["Content-Type"] = upload_request._content_type # actually ignored...
encoder = MultipartEncoder((rf_bin,))
try:
ri = self._get_api_request_invoker(c_path)
resp = ri.post(url=url, headers={"Content-Type": encoder.content_type}, data=encoder) # multipart/form-data
finally:
in_stream.close()
def _add_multipart(parts):
mime_multipart_parts = list()
for name, (filename, data, content_type) in parts.items():
multipart_part = RequestField(name=name, data=data, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
xml_request, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return xml_request, content_type
def _add_multipart(parts):
mime_multipart_parts = list()
for name, (filename, data, content_type) in parts.items():
multipart_part = RequestField(name=name, data=data, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
xml_request, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return xml_request, content_type
def add(self, document):
"""Add a document to the list of bulk uploads"""
if not isinstance(document, Documents):
raise InvalidAPIRequest(
"You can only pass documents to bulkloader")
if document.content() is None:
raise InvalidAPIRequest("You must specify the document content")
target = document.uris()
if len(target) != 1:
raise InvalidAPIRequest("You must specify a single URI")
else:
target = target[0]
self.field_count += 1
metaname = "meta{}".format(self.field_count)
dataname = "data{}".format(self.field_count)
self.logger.debug("Bulk[{}] = {}".format(self.field_count, target))
rf = RequestField(name=metaname,
data=document.metadata(),
filename=target)
rf.make_multipart(content_disposition='attachment; category=metadata', \
content_type=document.metadata_content_type())
self.fields.append(rf)
rf = RequestField(name=dataname,
data=document.content(),
filename=target)
rf.make_multipart(content_disposition='attachment', \
content_type=document.content_type())
self.fields.append(rf)
def _make_multipart(parts):
"""
Creates one "chunk" for a multi-part upload
'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).
Returns the post body and the content type string.
For more information, see this post:
http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests
"""
mime_multipart_parts = []
for name, (filename, blob, content_type) in parts.items():
multipart_part = RequestField(name=name, data=blob, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return post_body, content_type
def _make_multipart(parts):
"""
Creates one "chunk" for a multi-part upload
'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).
Returns the post body and the content type string.
"""
mime_multipart_parts = []
for name, (filename, blob, content_type) in parts.items():
multipart_part = RequestField(name=name, data=blob, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
return post_body, content_type
def create_attachment(ox, task):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
url = ox._url('attachment', 'attach')
params = ox._params()
json_0 = {'module': task.module_type,
'attached': task.id,
'folder': task.folder_id}
fields = []
rf = RequestField(name='json_0',data=json.dumps(json_0))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
rf = RequestField(name='file_0', data="TEXT", filename='attachment.txt')
rf.make_multipart(content_disposition='form-data',content_type='text/plain')
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body)
if response and response.status_code == 200:
regex='\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def add(self, document):
"""Add a document to the list of bulk uploads"""
if not isinstance(document, Documents):
raise InvalidAPIRequest("You can only pass documents to bulkloader")
if document.content() is None:
raise InvalidAPIRequest("You must specify the document content")
target = document.uris()
if len(target) != 1:
raise InvalidAPIRequest("You must specify a single URI")
else:
target = target[0]
self.field_count += 1
metaname = "meta{}".format(self.field_count)
dataname = "data{}".format(self.field_count)
self.logger.debug("Bulk[{}] = {}".format(self.field_count, target))
rf = RequestField(name=metaname, data=document.metadata(), filename=target)
rf.make_multipart(content_disposition='attachment; category=metadata', \
content_type=document.metadata_content_type())
self.fields.append(rf)
rf = RequestField(name=dataname, data=document.content(), filename=target)
rf.make_multipart(content_disposition='attachment', \
content_type=document.content_type())
self.fields.append(rf)
def _do_upload(self, upload_request):
c_path = upload_request.path
base_name = c_path.base_name()
parent_c_path = c_path.parent()
cm_root = self._load_folders_structure()
cm_parent_folder = cm_root.get_folder(parent_c_path)
if not cm_parent_folder:
# parent folder of given path does exist => folders needs to be created
cm_parent_folder = self._create_intermediary_folders(
cm_root, parent_c_path)
# Check: is it an existing folder ?
cm_folder = cm_parent_folder.get_child_by_name(base_name)
if cm_folder:
# The CPath corresponds to an existing folder, upload is not possible
raise CInvalidFileTypeError(c_path, True)
url = self._build_rest_url('documents') + cm_parent_folder.file_id
in_stream = upload_request.byte_source().open_stream()
rf_bin = RequestField(name='bin',
data=in_stream,
filename=c_path.base_name())
# Cloudme does not support UTF-8 encoded filenames that look like:
# filename*=UTF-8''...
# Instead we send raw UTF-8 bytes in between quotes (filename cannot contain any quote)
rf_bin.headers[
'Content-Disposition'] = 'form-data; name="bin"; filename="%s"' % base_name
rf_bin.headers[
'Content-Type'] = upload_request._content_type # actually ignored...
encoder = MultipartEncoder((rf_bin, ))
try:
ri = self._get_api_request_invoker(c_path)
resp = ri.post(
url=url,
headers={'Content-Type':
encoder.content_type}, # multipart/form-data
data=encoder)
finally:
in_stream.close()
def _make_multipart(parts):
"""
Creates one "chunk" for a multi-part upload.
'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).
Returns the post body and the content type string.
For more information, see this post:
http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests
"""
mime_multipart_parts = []
for name, (filename, blob, content_type) in parts.items():
multipart_part = RequestField(name=name, data=blob, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return post_body, content_type
def _put_mixed(self, data, target, connection):
"""
Put the document using the bulk interface (because it has
arbitrary metadata).
"""
params = []
for key in ['database', 'transform', 'txid', \
'temporal-collection', 'system-time']:
if key in self._config:
params.append("{}={}".format(key, self._config[key]))
for pair in self.transparams:
params.append("trans:{}={}".format(pair[0], pair[1]))
meta = self.metadata()
if self.metadata_content_type() is None:
metact = "application/xml"
else:
metact = self.metadata_content_type()
uri = connection.client_uri("documents")
if params:
uri = uri + "?" + "&".join(params)
datact = self._config['content-type']
fields = []
rf = RequestField(name="meta1", data=meta, filename=target)
rf.make_multipart(content_disposition='attachment; category=metadata', \
content_type=metact)
fields.append(rf)
rf = RequestField(name="data1", data=data, filename=target)
rf.make_multipart(content_disposition='attachment', \
content_type=datact)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
post_ct = ''.join(('multipart/mixed',) \
+ content_type.partition(';')[1:])
response = connection.post(uri,
payload=post_body,
content_type=post_ct)
return response
def create_attachment(ox, task):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
url = ox._url('attachment', 'attach')
params = ox._params()
json_0 = {
'module': task.module_type,
'attached': task.id,
'folder': task.folder_id
}
fields = []
rf = RequestField(name='json_0', data=json.dumps(json_0))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
rf = RequestField(name='file_0', data="TEXT", filename='attachment.txt')
rf.make_multipart(content_disposition='form-data',
content_type='text/plain')
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url,
cookies=ox._cookies,
params=params,
headers=headers,
data=post_body)
if response and response.status_code == 200:
regex = '\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def _compose_multipart(json_dict, filesdata):
""" Composes multipart/mixed request for create/edit brain.
The multipart message is constructed as 1st part application/json and
subsequent file part(s).
:param json: dictionary that will be json-encoded
:param filesdata: dict of <filename> -> <filedata>
"""
# requests 1.13 does not have high-level support for multipart/mixed.
# Using lower-level modules to construct multipart/mixed per
# http://stackoverflow.com/questions/26299889/
# how-to-post-multipart-list-of-json-xml-files-using-python-requests
fields = []
# 1st part: application/json
rf = RequestField(name="project_data", data=json.dumps(json_dict))
rf.make_multipart(content_type='application/json')
fields.append(rf)
# Subsequent parts: file text
for filename, filedata in filesdata.items():
rf = RequestField(name=filename,
data=filedata,
filename=filename,
headers={'Content-Length': len(filedata)})
rf.make_multipart(content_disposition='attachment',
content_type="application/octet-stream")
fields.append(rf)
# Compose message
body, content_type = encode_multipart_formdata(fields)
# "multipart/form-data; boundary=.." -> "multipart/mixed; boundary=.."
content_type = content_type.replace("multipart/form-data",
"multipart/mixed", 1)
headers = {'Content-Type': content_type}
return (headers, body)
def _put_mixed(self, data, target, connection):
"""
Put the document using the bulk interface (because it has
arbitrary metadata).
"""
params = []
for key in ["database", "transform", "txid", "temporal-collection", "system-time"]:
if key in self._config:
params.append("{}={}".format(key, self._config[key]))
for pair in self.transparams:
params.append("trans:{}={}".format(pair[0], pair[1]))
meta = self.metadata()
if self.metadata_content_type() is None:
metact = "application/xml"
else:
metact = self.metadata_content_type()
uri = connection.client_uri("documents")
if params:
uri = uri + "?" + "&".join(params)
datact = self._config["content-type"]
fields = []
rf = RequestField(name="meta1", data=meta, filename=target)
rf.make_multipart(content_disposition="attachment; category=metadata", content_type=metact)
fields.append(rf)
rf = RequestField(name="data1", data=data, filename=target)
rf.make_multipart(content_disposition="attachment", content_type=datact)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
post_ct = "".join(("multipart/mixed",) + content_type.partition(";")[1:])
response = connection.post(uri, payload=post_body, content_type=post_ct)
return response
def upload(bean, args=[{'content':None,'file':None, 'mimetype':'text/plain','name':'attachment.txt'}]):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
ox = bean._ox
url = ox._url('attachment', 'attach')
params = ox._params()
meta = {'module': bean.module_type,
#'attached': bean.id,
'folder': bean.folder_id}
counter = 0; fields = []
for data in args:
# json metadata
rf = RequestField(name='json_' + str(counter) ,data=json.dumps(meta))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
# content: data or file to read
filename = 'attachment.txt'
mimetype = 'text/plain'
content = None
if 'content' in data:
content = data['content']
else:
if 'file' in data:
filename = data['file']
if os.path.isfile(filename):
with open(filename, 'rb') as fh:
content = fh.read()
if content is None:
#TODO: process error
return None
if 'name' in data:
filename = data['name']
mimetype = 'text/plain'
if 'mimetype' in data:
mimetype = data['mimetype']
rf = RequestField(name='file_' + str(counter), data=content, filename=filename)
rf.make_multipart(content_disposition='form-data',content_type=mimetype)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body)
if response and response.status_code == 200:
regex='\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def upload(bean,
args=[{
'content': None,
'file': None,
'mimetype': 'text/plain',
'name': 'attachment.txt'
}]):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
ox = bean._ox
url = ox._url('attachment', 'attach')
params = ox._params()
meta = {
'module': bean.module_type,
#'attached': bean.id,
'folder': bean.folder_id
}
counter = 0
fields = []
for data in args:
# json metadata
rf = RequestField(name='json_' + str(counter), data=json.dumps(meta))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
# content: data or file to read
filename = 'attachment.txt'
mimetype = 'text/plain'
content = None
if 'content' in data:
content = data['content']
else:
if 'file' in data:
filename = data['file']
if os.path.isfile(filename):
with open(filename, 'rb') as fh:
content = fh.read()
if content is None:
#TODO: process error
return None
if 'name' in data:
filename = data['name']
mimetype = 'text/plain'
if 'mimetype' in data:
mimetype = data['mimetype']
rf = RequestField(name='file_' + str(counter),
data=content,
filename=filename)
rf.make_multipart(content_disposition='form-data',
content_type=mimetype)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url,
cookies=ox._cookies,
params=params,
headers=headers,
data=post_body)
if response and response.status_code == 200:
regex = '\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def _do_upload(self, upload_request):
# Check before upload: is it a folder ?
# (uploading a blob would create another file with the same name: bad)
c_path = upload_request.path
remote_path = self._find_remote_path(c_path)
if remote_path.exists() and not remote_path.last_is_blob():
# path refer to an existing folder: wrong !
raise CInvalidFileTypeError(c_path, True)
if not remote_path.exists() and remote_path.last_is_blob():
# some blob exists in path: wrong !
raise CInvalidFileTypeError(remote_path.last_c_path(), False)
# only one of these 2 will be set:
file_id = None
parent_id = None
if remote_path.exists():
# Blob already exists: we'll update it
file_id = remote_path.get_blob()['id']
else:
parent_id = remote_path.deepest_folder_id()
# We may need to create intermediary folders first:
# create intermediary folders, in case:
i = len(remote_path.files_chain)
while i < len(remote_path.segments) - 1:
current_c_path = remote_path.first_segments_path(i + 1)
parent_id = self._raw_create_folder(current_c_path, parent_id)
i += 1
# By now we can upload a new blob to folder with id=parent_id,
# or update existing blob with id=file_id:
# TODO handle metadata:
# if upload_request._medatada:
ri = self._get_api_request_invoker(c_path)
if file_id:
# Blob update
json_meta = {}
else:
# Blob creation
json_meta = {
'title': c_path.base_name(),
'parents': [{
'id': parent_id
}]
}
if upload_request._content_type:
# It seems that drive distinguishes between mimeType defined here,
# and Content-Type defined in part header.
# Drive also tries to guess mimeType...
json_meta[
'mimeType'] = upload_request._content_type # seems to be ignored ? mimeType not updatable ?
in_stream = upload_request.byte_source().open_stream()
fields = (RequestField(
name=None,
data=json.dumps(json_meta),
filename=None,
headers={'Content-Type': 'application/json; charset=UTF-8'}),
RequestField(
name=None,
data=in_stream,
filename=None,
headers={'Content-Type': upload_request._content_type}))
encoder = MultipartEncoder(fields, 'related')
try:
if file_id:
# Updating existing file:
resp = ri.put(
self.FILES_UPLOAD_ENDPOINT + '/' + file_id,
headers={'Content-Type':
encoder.content_type}, # multipart/related
params={'uploadType': 'multipart'},
data=encoder)
else:
# uploading a new file:
resp = ri.post(
self.FILES_UPLOAD_ENDPOINT,
headers={'Content-Type':
encoder.content_type}, # multipart/related
params={'uploadType': 'multipart'},
data=encoder)
finally:
in_stream.close()
