def add(self, document): """Add a document to the list of bulk uploads""" if not isinstance(document, Documents): raise InvalidAPIRequest( "You can only pass documents to bulkloader") if document.content() is None: raise InvalidAPIRequest("You must specify the document content") target = document.uris() if len(target) != 1: raise InvalidAPIRequest("You must specify a single URI") else: target = target[0] self.field_count += 1 metaname = "meta{}".format(self.field_count) dataname = "data{}".format(self.field_count) self.logger.debug("Bulk[{}] = {}".format(self.field_count, target)) rf = RequestField(name=metaname, data=document.metadata(), filename=target) rf.make_multipart(content_disposition='attachment; category=metadata', \ content_type=document.metadata_content_type()) self.fields.append(rf) rf = RequestField(name=dataname, data=document.content(), filename=target) rf.make_multipart(content_disposition='attachment', \ content_type=document.content_type()) self.fields.append(rf)
def create_attachment(ox, task): from requests.packages.urllib3.fields import RequestField from requests.packages.urllib3.filepost import encode_multipart_formdata url = ox._url('attachment', 'attach') params = ox._params() json_0 = {'module': task.module_type, 'attached': task.id, 'folder': task.folder_id} fields = [] rf = RequestField(name='json_0',data=json.dumps(json_0)) rf.make_multipart(content_disposition='form-data') fields.append(rf) rf = RequestField(name='file_0', data="TEXT", filename='attachment.txt') rf.make_multipart(content_disposition='form-data',content_type='text/plain') fields.append(rf) post_body, content_type = encode_multipart_formdata(fields) content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:]) headers = {'Content-Type': content_type} response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body) if response and response.status_code == 200: regex='\((\{.*\})\)' match = re.search(regex, response.content) if match: return json.loads(match.group(1)) return None
def add(self, document): """Add a document to the list of bulk uploads""" if not isinstance(document, Documents): raise InvalidAPIRequest("You can only pass documents to bulkloader") if document.content() is None: raise InvalidAPIRequest("You must specify the document content") target = document.uris() if len(target) != 1: raise InvalidAPIRequest("You must specify a single URI") else: target = target[0] self.field_count += 1 metaname = "meta{}".format(self.field_count) dataname = "data{}".format(self.field_count) self.logger.debug("Bulk[{}] = {}".format(self.field_count, target)) rf = RequestField(name=metaname, data=document.metadata(), filename=target) rf.make_multipart(content_disposition='attachment; category=metadata', \ content_type=document.metadata_content_type()) self.fields.append(rf) rf = RequestField(name=dataname, data=document.content(), filename=target) rf.make_multipart(content_disposition='attachment', \ content_type=document.content_type()) self.fields.append(rf)
def _iter_field(cls, field_name, field): file_name = None file_type = None file_headers = None if field and isinstance(field, (list, tuple)): if all([isinstance(f, (list, tuple)) for f in field]): for f in field: yield next(cls._iter_field(field_name, f)) else: raise StopIteration() if len(field) == 2: file_name, file_pointer = field elif len(field) == 3: file_name, file_pointer, file_type = field else: file_name, file_pointer, file_type, file_headers = field else: file_pointer = field field = RequestField(name=field_name, data=file_pointer, filename=file_name, headers=file_headers) field.make_multipart(content_type=file_type) yield field
def _add_multipart(parts): mime_multipart_parts = list() for name, (filename, data, content_type) in parts.items(): multipart_part = RequestField(name=name, data=data, filename=filename) multipart_part.make_multipart(content_type=content_type) mime_multipart_parts.append(multipart_part) xml_request, content_type = encode_multipart_formdata(mime_multipart_parts) content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:]) return xml_request, content_type
def _add_multipart(parts): mime_multipart_parts = list() for name, (filename, data, content_type) in parts.items(): multipart_part = RequestField(name=name, data=data, filename=filename) multipart_part.make_multipart(content_type=content_type) mime_multipart_parts.append(multipart_part) xml_request, content_type = encode_multipart_formdata(mime_multipart_parts) content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:]) return xml_request, content_type
def _put_mixed(self, data, target, connection): """ Put the document using the bulk interface (because it has arbitrary metadata). """ params = [] for key in ['database', 'transform', 'txid', \ 'temporal-collection', 'system-time']: if key in self._config: params.append("{}={}".format(key, self._config[key])) for pair in self.transparams: params.append("trans:{}={}".format(pair[0], pair[1])) meta = self.metadata() if self.metadata_content_type() is None: metact = "application/xml" else: metact = self.metadata_content_type() uri = connection.client_uri("documents") if params: uri = uri + "?" + "&".join(params) datact = self._config['content-type'] fields = [] rf = RequestField(name="meta1", data=meta, filename=target) rf.make_multipart(content_disposition='attachment; category=metadata', \ content_type=metact) fields.append(rf) rf = RequestField(name="data1", data=data, filename=target) rf.make_multipart(content_disposition='attachment', \ content_type=datact) fields.append(rf) post_body, content_type = encode_multipart_formdata(fields) post_ct = ''.join(('multipart/mixed',) \ + content_type.partition(';')[1:]) response = connection.post(uri, payload=post_body, content_type=post_ct) return response
def _make_multipart(parts): """ Creates one "chunk" for a multi-part upload 'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type). Returns the post body and the content type string. For more information, see this post: http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests """ mime_multipart_parts = [] for name, (filename, blob, content_type) in parts.items(): multipart_part = RequestField(name=name, data=blob, filename=filename) multipart_part.make_multipart(content_type=content_type) mime_multipart_parts.append(multipart_part) post_body, content_type = encode_multipart_formdata(mime_multipart_parts) content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:]) return post_body, content_type
def _make_multipart(parts): """ Creates one "chunk" for a multi-part upload 'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type). Returns the post body and the content type string. """ mime_multipart_parts = [] for name, (filename, blob, content_type) in parts.items(): multipart_part = RequestField(name=name, data=blob, filename=filename) multipart_part.make_multipart(content_type=content_type) mime_multipart_parts.append(multipart_part) post_body, content_type = encode_multipart_formdata(mime_multipart_parts) content_type = ''.join(('multipart/mixed', ) + content_type.partition(';')[1:]) return post_body, content_type
def create_attachment(ox, task): from requests.packages.urllib3.fields import RequestField from requests.packages.urllib3.filepost import encode_multipart_formdata url = ox._url('attachment', 'attach') params = ox._params() json_0 = { 'module': task.module_type, 'attached': task.id, 'folder': task.folder_id } fields = [] rf = RequestField(name='json_0', data=json.dumps(json_0)) rf.make_multipart(content_disposition='form-data') fields.append(rf) rf = RequestField(name='file_0', data="TEXT", filename='attachment.txt') rf.make_multipart(content_disposition='form-data', content_type='text/plain') fields.append(rf) post_body, content_type = encode_multipart_formdata(fields) content_type = ''.join(('multipart/mixed', ) + content_type.partition(';')[1:]) headers = {'Content-Type': content_type} response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body) if response and response.status_code == 200: regex = '\((\{.*\})\)' match = re.search(regex, response.content) if match: return json.loads(match.group(1)) return None
def _put_mixed(self, data, target, connection): """ Put the document using the bulk interface (because it has arbitrary metadata). """ params = [] for key in ["database", "transform", "txid", "temporal-collection", "system-time"]: if key in self._config: params.append("{}={}".format(key, self._config[key])) for pair in self.transparams: params.append("trans:{}={}".format(pair[0], pair[1])) meta = self.metadata() if self.metadata_content_type() is None: metact = "application/xml" else: metact = self.metadata_content_type() uri = connection.client_uri("documents") if params: uri = uri + "?" + "&".join(params) datact = self._config["content-type"] fields = [] rf = RequestField(name="meta1", data=meta, filename=target) rf.make_multipart(content_disposition="attachment; category=metadata", content_type=metact) fields.append(rf) rf = RequestField(name="data1", data=data, filename=target) rf.make_multipart(content_disposition="attachment", content_type=datact) fields.append(rf) post_body, content_type = encode_multipart_formdata(fields) post_ct = "".join(("multipart/mixed",) + content_type.partition(";")[1:]) response = connection.post(uri, payload=post_body, content_type=post_ct) return response
def _make_multipart(parts): """ Creates one "chunk" for a multi-part upload. 'parts' is a dictionary that provides key-value pairs of the format name: (filename, body, content_type). Returns the post body and the content type string. For more information, see this post: http://stackoverflow.com/questions/26299889/how-to-post-multipart-list-of-json-xml-files-using-python-requests """ mime_multipart_parts = [] for name, (filename, blob, content_type) in parts.items(): multipart_part = RequestField(name=name, data=blob, filename=filename) multipart_part.make_multipart(content_type=content_type) mime_multipart_parts.append(multipart_part) post_body, content_type = encode_multipart_formdata(mime_multipart_parts) content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:]) return post_body, content_type
def _compose_multipart(json_dict, filesdata): """ Composes multipart/mixed request for create/edit brain. The multipart message is constructed as 1st part application/json and subsequent file part(s). :param json: dictionary that will be json-encoded :param filesdata: dict of <filename> -> <filedata> """ # requests 1.13 does not have high-level support for multipart/mixed. # Using lower-level modules to construct multipart/mixed per # http://stackoverflow.com/questions/26299889/ # how-to-post-multipart-list-of-json-xml-files-using-python-requests fields = [] # 1st part: application/json rf = RequestField(name="project_data", data=json.dumps(json_dict)) rf.make_multipart(content_type='application/json') fields.append(rf) # Subsequent parts: file text for filename, filedata in filesdata.items(): rf = RequestField(name=filename, data=filedata, filename=filename, headers={'Content-Length': len(filedata)}) rf.make_multipart(content_disposition='attachment', content_type="application/octet-stream") fields.append(rf) # Compose message body, content_type = encode_multipart_formdata(fields) # "multipart/form-data; boundary=.." -> "multipart/mixed; boundary=.." content_type = content_type.replace("multipart/form-data", "multipart/mixed", 1) headers = {'Content-Type': content_type} return (headers, body)
def upload(bean, args=[{ 'content': None, 'file': None, 'mimetype': 'text/plain', 'name': 'attachment.txt' }]): from requests.packages.urllib3.fields import RequestField from requests.packages.urllib3.filepost import encode_multipart_formdata ox = bean._ox url = ox._url('attachment', 'attach') params = ox._params() meta = { 'module': bean.module_type, #'attached': bean.id, 'folder': bean.folder_id } counter = 0 fields = [] for data in args: # json metadata rf = RequestField(name='json_' + str(counter), data=json.dumps(meta)) rf.make_multipart(content_disposition='form-data') fields.append(rf) # content: data or file to read filename = 'attachment.txt' mimetype = 'text/plain' content = None if 'content' in data: content = data['content'] else: if 'file' in data: filename = data['file'] if os.path.isfile(filename): with open(filename, 'rb') as fh: content = fh.read() if content is None: #TODO: process error return None if 'name' in data: filename = data['name'] mimetype = 'text/plain' if 'mimetype' in data: mimetype = data['mimetype'] rf = RequestField(name='file_' + str(counter), data=content, filename=filename) rf.make_multipart(content_disposition='form-data', content_type=mimetype) fields.append(rf) post_body, content_type = encode_multipart_formdata(fields) content_type = ''.join(('multipart/mixed', ) + content_type.partition(';')[1:]) headers = {'Content-Type': content_type} response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body) if response and response.status_code == 200: regex = '\((\{.*\})\)' match = re.search(regex, response.content) if match: return json.loads(match.group(1)) return None
def upload(bean, args=[{'content':None,'file':None, 'mimetype':'text/plain','name':'attachment.txt'}]): from requests.packages.urllib3.fields import RequestField from requests.packages.urllib3.filepost import encode_multipart_formdata ox = bean._ox url = ox._url('attachment', 'attach') params = ox._params() meta = {'module': bean.module_type, #'attached': bean.id, 'folder': bean.folder_id} counter = 0; fields = [] for data in args: # json metadata rf = RequestField(name='json_' + str(counter) ,data=json.dumps(meta)) rf.make_multipart(content_disposition='form-data') fields.append(rf) # content: data or file to read filename = 'attachment.txt' mimetype = 'text/plain' content = None if 'content' in data: content = data['content'] else: if 'file' in data: filename = data['file'] if os.path.isfile(filename): with open(filename, 'rb') as fh: content = fh.read() if content is None: #TODO: process error return None if 'name' in data: filename = data['name'] mimetype = 'text/plain' if 'mimetype' in data: mimetype = data['mimetype'] rf = RequestField(name='file_' + str(counter), data=content, filename=filename) rf.make_multipart(content_disposition='form-data',content_type=mimetype) fields.append(rf) post_body, content_type = encode_multipart_formdata(fields) content_type = ''.join(('multipart/mixed',) + content_type.partition(';')[1:]) headers = {'Content-Type': content_type} response = requests.post(url, cookies=ox._cookies, params=params, headers=headers, data=post_body) if response and response.status_code == 200: regex='\((\{.*\})\)' match = re.search(regex, response.content) if match: return json.loads(match.group(1)) return None