def add(self, document):
"""Add a document to the list of bulk uploads"""
if not isinstance(document, Documents):
raise InvalidAPIRequest(
"You can only pass documents to bulkloader")
if document.content() is None:
raise InvalidAPIRequest("You must specify the document content")
target = document.uris()
if len(target) != 1:
raise InvalidAPIRequest("You must specify a single URI")
else:
target = target[0]
self.field_count += 1
metaname = "meta{}".format(self.field_count)
dataname = "data{}".format(self.field_count)
self.logger.debug("Bulk[{}] = {}".format(self.field_count, target))
rf = RequestField(name=metaname,
data=document.metadata(),
filename=target)
rf.make_multipart(content_disposition='attachment; category=metadata', \
content_type=document.metadata_content_type())
self.fields.append(rf)
rf = RequestField(name=dataname,
data=document.content(),
filename=target)
rf.make_multipart(content_disposition='attachment', \
content_type=document.content_type())
self.fields.append(rf)
def create_attachment(ox, task):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
url = ox._url('attachment', 'attach')
params = ox._params()
json_0 = {'module': task.module_type,
'attached': task.id,
'folder': task.folder_id}
fields = []
rf = RequestField(name='json_0',data=json.dumps(json_0))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
rf = RequestField(name='file_0', data="TEXT", filename='attachment.txt')
rf.make_multipart(content_disposition='form-data',content_type='text/plain')
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body)
if response and response.status_code == 200:
regex='\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def add(self, document):
"""Add a document to the list of bulk uploads"""
if not isinstance(document, Documents):
raise InvalidAPIRequest("You can only pass documents to bulkloader")
if document.content() is None:
raise InvalidAPIRequest("You must specify the document content")
target = document.uris()
if len(target) != 1:
raise InvalidAPIRequest("You must specify a single URI")
else:
target = target[0]
self.field_count += 1
metaname = "meta{}".format(self.field_count)
dataname = "data{}".format(self.field_count)
self.logger.debug("Bulk[{}] = {}".format(self.field_count, target))
rf = RequestField(name=metaname, data=document.metadata(), filename=target)
rf.make_multipart(content_disposition='attachment; category=metadata', \
content_type=document.metadata_content_type())
self.fields.append(rf)
rf = RequestField(name=dataname, data=document.content(), filename=target)
rf.make_multipart(content_disposition='attachment', \
content_type=document.content_type())
self.fields.append(rf)
def _iter_field(cls, field_name, field):
file_name = None
file_type = None
file_headers = None
if field and isinstance(field, (list, tuple)):
if all([isinstance(f, (list, tuple)) for f in field]):
for f in field:
yield next(cls._iter_field(field_name, f))
else:
raise StopIteration()
if len(field) == 2:
file_name, file_pointer = field
elif len(field) == 3:
file_name, file_pointer, file_type = field
else:
file_name, file_pointer, file_type, file_headers = field
else:
file_pointer = field
field = RequestField(name=field_name,
data=file_pointer,
filename=file_name,
headers=file_headers)
field.make_multipart(content_type=file_type)
yield field
def _add_multipart(parts):
mime_multipart_parts = list()
for name, (filename, data, content_type) in parts.items():
multipart_part = RequestField(name=name, data=data, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
xml_request, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return xml_request, content_type
def _add_multipart(parts):
mime_multipart_parts = list()
for name, (filename, data, content_type) in parts.items():
multipart_part = RequestField(name=name, data=data, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
xml_request, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return xml_request, content_type
def _put_mixed(self, data, target, connection):
"""
Put the document using the bulk interface (because it has
arbitrary metadata).
"""
params = []
for key in ['database', 'transform', 'txid', \
'temporal-collection', 'system-time']:
if key in self._config:
params.append("{}={}".format(key, self._config[key]))
for pair in self.transparams:
params.append("trans:{}={}".format(pair[0], pair[1]))
meta = self.metadata()
if self.metadata_content_type() is None:
metact = "application/xml"
else:
metact = self.metadata_content_type()
uri = connection.client_uri("documents")
if params:
uri = uri + "?" + "&".join(params)
datact = self._config['content-type']
fields = []
rf = RequestField(name="meta1", data=meta, filename=target)
rf.make_multipart(content_disposition='attachment; category=metadata', \
content_type=metact)
fields.append(rf)
rf = RequestField(name="data1", data=data, filename=target)
rf.make_multipart(content_disposition='attachment', \
content_type=datact)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
post_ct = ''.join(('multipart/mixed',) \
+ content_type.partition(';')[1:])
response = connection.post(uri,
payload=post_body,
content_type=post_ct)
return response
def _make_multipart(parts):
"""
Creates one "chunk" for a multi-part upload
'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).
Returns the post body and the content type string.
For more information, see this post:
http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests
"""
mime_multipart_parts = []
for name, (filename, blob, content_type) in parts.items():
multipart_part = RequestField(name=name, data=blob, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return post_body, content_type
def _make_multipart(parts):
"""
Creates one "chunk" for a multi-part upload
'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).
Returns the post body and the content type string.
"""
mime_multipart_parts = []
for name, (filename, blob, content_type) in parts.items():
multipart_part = RequestField(name=name, data=blob, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
return post_body, content_type
def create_attachment(ox, task):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
url = ox._url('attachment', 'attach')
params = ox._params()
json_0 = {
'module': task.module_type,
'attached': task.id,
'folder': task.folder_id
}
fields = []
rf = RequestField(name='json_0', data=json.dumps(json_0))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
rf = RequestField(name='file_0', data="TEXT", filename='attachment.txt')
rf.make_multipart(content_disposition='form-data',
content_type='text/plain')
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url,
cookies=ox._cookies,
params=params,
headers=headers,
data=post_body)
if response and response.status_code == 200:
regex = '\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def _put_mixed(self, data, target, connection):
"""
Put the document using the bulk interface (because it has
arbitrary metadata).
"""
params = []
for key in ["database", "transform", "txid", "temporal-collection", "system-time"]:
if key in self._config:
params.append("{}={}".format(key, self._config[key]))
for pair in self.transparams:
params.append("trans:{}={}".format(pair[0], pair[1]))
meta = self.metadata()
if self.metadata_content_type() is None:
metact = "application/xml"
else:
metact = self.metadata_content_type()
uri = connection.client_uri("documents")
if params:
uri = uri + "?" + "&".join(params)
datact = self._config["content-type"]
fields = []
rf = RequestField(name="meta1", data=meta, filename=target)
rf.make_multipart(content_disposition="attachment; category=metadata", content_type=metact)
fields.append(rf)
rf = RequestField(name="data1", data=data, filename=target)
rf.make_multipart(content_disposition="attachment", content_type=datact)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
post_ct = "".join(("multipart/mixed",) + content_type.partition(";")[1:])
response = connection.post(uri, payload=post_body, content_type=post_ct)
return response
def _make_multipart(parts):
"""
Creates one "chunk" for a multi-part upload.
'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type).
Returns the post body and the content type string.
For more information, see this post:
http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests
"""
mime_multipart_parts = []
for name, (filename, blob, content_type) in parts.items():
multipart_part = RequestField(name=name, data=blob, filename=filename)
multipart_part.make_multipart(content_type=content_type)
mime_multipart_parts.append(multipart_part)
post_body, content_type = encode_multipart_formdata(mime_multipart_parts)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
return post_body, content_type
def _compose_multipart(json_dict, filesdata):
""" Composes multipart/mixed request for create/edit brain.
The multipart message is constructed as 1st part application/json and
subsequent file part(s).
:param json: dictionary that will be json-encoded
:param filesdata: dict of <filename> -> <filedata>
"""
# requests 1.13 does not have high-level support for multipart/mixed.
# Using lower-level modules to construct multipart/mixed per
# http://stackoverflow.com/questions/26299889/
# how-to-post-multipart-list-of-json-xml-files-using-python-requests
fields = []
# 1st part: application/json
rf = RequestField(name="project_data", data=json.dumps(json_dict))
rf.make_multipart(content_type='application/json')
fields.append(rf)
# Subsequent parts: file text
for filename, filedata in filesdata.items():
rf = RequestField(name=filename,
data=filedata,
filename=filename,
headers={'Content-Length': len(filedata)})
rf.make_multipart(content_disposition='attachment',
content_type="application/octet-stream")
fields.append(rf)
# Compose message
body, content_type = encode_multipart_formdata(fields)
# "multipart/form-data; boundary=.." -> "multipart/mixed; boundary=.."
content_type = content_type.replace("multipart/form-data",
"multipart/mixed", 1)
headers = {'Content-Type': content_type}
return (headers, body)
def upload(bean,
args=[{
'content': None,
'file': None,
'mimetype': 'text/plain',
'name': 'attachment.txt'
}]):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
ox = bean._ox
url = ox._url('attachment', 'attach')
params = ox._params()
meta = {
'module': bean.module_type,
#'attached': bean.id,
'folder': bean.folder_id
}
counter = 0
fields = []
for data in args:
# json metadata
rf = RequestField(name='json_' + str(counter), data=json.dumps(meta))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
# content: data or file to read
filename = 'attachment.txt'
mimetype = 'text/plain'
content = None
if 'content' in data:
content = data['content']
else:
if 'file' in data:
filename = data['file']
if os.path.isfile(filename):
with open(filename, 'rb') as fh:
content = fh.read()
if content is None:
#TODO: process error
return None
if 'name' in data:
filename = data['name']
mimetype = 'text/plain'
if 'mimetype' in data:
mimetype = data['mimetype']
rf = RequestField(name='file_' + str(counter),
data=content,
filename=filename)
rf.make_multipart(content_disposition='form-data',
content_type=mimetype)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed', ) +
content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url,
cookies=ox._cookies,
params=params,
headers=headers,
data=post_body)
if response and response.status_code == 200:
regex = '\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
def upload(bean, args=[{'content':None,'file':None, 'mimetype':'text/plain','name':'attachment.txt'}]):
from requests.packages.urllib3.fields import RequestField
from requests.packages.urllib3.filepost import encode_multipart_formdata
ox = bean._ox
url = ox._url('attachment', 'attach')
params = ox._params()
meta = {'module': bean.module_type,
#'attached': bean.id,
'folder': bean.folder_id}
counter = 0; fields = []
for data in args:
# json metadata
rf = RequestField(name='json_' + str(counter) ,data=json.dumps(meta))
rf.make_multipart(content_disposition='form-data')
fields.append(rf)
# content: data or file to read
filename = 'attachment.txt'
mimetype = 'text/plain'
content = None
if 'content' in data:
content = data['content']
else:
if 'file' in data:
filename = data['file']
if os.path.isfile(filename):
with open(filename, 'rb') as fh:
content = fh.read()
if content is None:
#TODO: process error
return None
if 'name' in data:
filename = data['name']
mimetype = 'text/plain'
if 'mimetype' in data:
mimetype = data['mimetype']
rf = RequestField(name='file_' + str(counter), data=content, filename=filename)
rf.make_multipart(content_disposition='form-data',content_type=mimetype)
fields.append(rf)
post_body, content_type = encode_multipart_formdata(fields)
content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:])
headers = {'Content-Type': content_type}
response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body)
if response and response.status_code == 200:
regex='\((\{.*\})\)'
match = re.search(regex, response.content)
if match:
return json.loads(match.group(1))
return None
