def upload(self, batch_id, file_path, filename=None, file_index=0, mime_type=None): """Upload a file through an Automation batch Uses poster.httpstreaming to stream the upload and not load the whole file in memory. """ FileAction("Upload", file_path, filename) # Request URL if self.is_new_upload_api_available(): url = self.rest_api_url + self.batch_upload_path + '/' + batch_id + '/' + str(file_index) else: # Backward compatibility with old batch upload API url = self.automation_url.encode('ascii') + self.batch_upload_url # HTTP headers if filename is None: filename = os.path.basename(file_path) file_size = os.path.getsize(file_path) if mime_type is None: mime_type = guess_mime_type(filename) # Quote UTF-8 filenames even though JAX-RS does not seem to be able # to retrieve them as per: https://tools.ietf.org/html/rfc5987 filename = safe_filename(filename) quoted_filename = urllib2.quote(filename.encode('utf-8')) headers = { "X-File-Name": quoted_filename, "X-File-Size": file_size, "X-File-Type": mime_type, "Content-Type": "application/octet-stream", "Content-Length": file_size, } if not self.is_new_upload_api_available(): headers.update({"X-Batch-Id": batch_id, "X-File-Idx": file_index}) headers.update(self._get_common_headers()) # Request data input_file = open(file_path, 'rb') # Use file system block size if available for streaming buffer fs_block_size = self.get_upload_buffer(input_file) data = self._read_data(input_file, fs_block_size) # Execute request cookies = self._get_cookies() log.trace("Calling %s with headers %r and cookies %r for file %s", url, headers, cookies, file_path) req = urllib2.Request(url, data, headers) try: resp = self.streaming_opener.open(req, timeout=self.blob_timeout) except Exception as e: log_details = self._log_details(e) if isinstance(log_details, tuple): _, _, _, error = log_details if error and error.startswith("Unable to find batch"): raise InvalidBatchException() raise e finally: input_file.close() self.end_action() return self._read_response(resp, url)
def upload(self, batch_id, file_path, filename=None, file_index=0, mime_type=None): """Upload a file through an Automation batch Uses poster.httpstreaming to stream the upload and not load the whole file in memory. """ self.current_action = FileAction("Upload", file_path, filename) # Request URL url = self.automation_url.encode('ascii') + self.batch_upload_url # HTTP headers if filename is None: filename = os.path.basename(file_path) file_size = os.path.getsize(file_path) if mime_type is None: mime_type = guess_mime_type(filename) # Quote UTF-8 filenames even though JAX-RS does not seem to be able # to retrieve them as per: https://tools.ietf.org/html/rfc5987 filename = safe_filename(filename) quoted_filename = urllib2.quote(filename.encode('utf-8')) headers = { "X-Batch-Id": batch_id, "X-File-Idx": file_index, "X-File-Name": quoted_filename, "X-File-Size": file_size, "X-File-Type": mime_type, "Content-Type": "application/octet-stream", "Content-Length": file_size, } headers.update(self._get_common_headers()) # Request data input_file = open(file_path, 'rb') # Use file system block size if available for streaming buffer fs_block_size = self.get_upload_buffer(input_file) log.trace( "Using file system block size" " for the streaming upload buffer: %u bytes", fs_block_size) data = self._read_data(input_file, fs_block_size) # Execute request cookies = self._get_cookies() log.trace("Calling %s with headers %r and cookies %r for file %s", url, headers, cookies, file_path) req = urllib2.Request(url, data, headers) try: resp = self.streaming_opener.open(req, timeout=self.blob_timeout) except Exception as e: self._log_details(e) raise finally: input_file.close() self.end_action() return self._read_response(resp, url)
def upload(self, batch_id, file_path, filename=None, file_index=0, mime_type=None): """Upload a file through an Automation batch Uses poster.httpstreaming to stream the upload and not load the whole file in memory. """ # Request URL url = self.automation_url.encode('ascii') + self.batch_upload_url # HTTP headers if filename is None: filename = os.path.basename(file_path) file_size = os.path.getsize(file_path) if mime_type is None: mime_type = guess_mime_type(filename) # Quote UTF-8 filenames even though JAX-RS does not seem to be able # to retrieve them as per: https://tools.ietf.org/html/rfc5987 filename = safe_filename(filename) quoted_filename = urllib2.quote(filename.encode('utf-8')) headers = { "X-Batch-Id": batch_id, "X-File-Idx": file_index, "X-File-Name": quoted_filename, "X-File-Size": file_size, "X-File-Type": mime_type, "Content-Type": "application/octet-stream", "Content-Length": file_size, } headers.update(self._get_common_headers()) # Request data input_file = open(file_path, 'rb') # Use file system block size if available for streaming buffer if sys.platform != 'win32': fs_block_size = os.fstatvfs(input_file.fileno()).f_bsize else: fs_block_size = FILE_BUFFER_SIZE log.trace("Using file system block size" " for the streaming upload buffer: %u bytes", fs_block_size) data = self._read_data(input_file, fs_block_size) # Execute request cookies = self._get_cookies() log.trace("Calling %s with headers %r and cookies %r for file %s", url, headers, cookies, file_path) req = urllib2.Request(url, data, headers) try: resp = self.streaming_opener.open(req, timeout=self.blob_timeout) except Exception as e: self._log_details(e) raise finally: input_file.close() return self._read_response(resp, url)
def test_guess_mime_type(self): # Text self.assertEquals(guess_mime_type('text.txt'), 'text/plain') self.assertEquals(guess_mime_type('text.html'), 'text/html') self.assertEquals(guess_mime_type('text.css'), 'text/css') self.assertEquals(guess_mime_type('text.js'), 'application/javascript') # Image self.assertEquals(guess_mime_type('picture.jpg'), 'image/jpeg') self.assertEquals(guess_mime_type('picture.png'), 'image/png') self.assertEquals(guess_mime_type('picture.gif'), 'image/gif') self.assertIn(guess_mime_type('picture.bmp'), ['image/x-ms-bmp', 'image/bmp']) self.assertEquals(guess_mime_type('picture.tiff'), 'image/tiff') self.assertIn(guess_mime_type('picture.ico'), ['image/x-icon', 'image/vnd.microsoft.icon']) # Audio self.assertEquals(guess_mime_type('sound.mp3'), 'audio/mpeg') self.assertIn(guess_mime_type('sound.wma'), ['audio/x-ms-wma', 'application/octet-stream']) self.assertEquals(guess_mime_type('sound.wav'), 'audio/x-wav') # Video self.assertEquals(guess_mime_type('video.mpeg'), 'video/mpeg') self.assertEquals(guess_mime_type('video.mp4'), 'video/mp4') self.assertEquals(guess_mime_type('video.mov'), 'video/quicktime') self.assertIn(guess_mime_type('video.wmv'), ['video/x-ms-wmv', 'application/octet-stream']) self.assertIn(guess_mime_type('video.avi'), ['video/x-msvideo', 'video/avi']) # Office self.assertEquals(guess_mime_type('office.doc'), 'application/msword') self.assertIn(guess_mime_type('office.xls'), ['application/vnd.ms-excel', 'application/x-msexcel']) self.assertIn( guess_mime_type('office.ppt'), ['application/vnd.ms-powerpoint', 'application/x-mspowerpoint']) # PDF self.assertEquals(guess_mime_type('document.pdf'), 'application/pdf') # Unknown self.assertEquals(guess_mime_type('file.unknown'), 'application/octet-stream') # Cases badly handled by Windows # See https://jira.nuxeo.com/browse/NXP-11660 # and http://bugs.python.org/issue15207 if sys.platform == "win32": # Text self.assertEquals(guess_mime_type('text.csv'), 'application/octet-stream') self.assertEquals(guess_mime_type('text.xml'), 'text/xml') # Image self.assertIn(guess_mime_type('picture.svg'), ['image/svg+xml', 'application/octet-stream']) # Video self.assertEquals(guess_mime_type('video.flv'), 'application/octet-stream') # Office self.assertIn(guess_mime_type('office.docx'), [ 'application/vnd.openxmlformats-officedocument' '.wordprocessingml.document', 'application/octet-stream' ]) self.assertIn(guess_mime_type('office.xlsx'), [ 'application/vnd.openxmlformats-officedocument' '.spreadsheetml.sheet', 'application/octet-stream' ]) self.assertIn(guess_mime_type('office.pptx'), [ 'application/vnd.openxmlformats-officedocument' '.presentationml.presentation', 'application/x-mspowerpoint.12', 'application/octet-stream' ]) self.assertIn(guess_mime_type('office.odt'), [ 'application/vnd.oasis.opendocument.text', 'application/octet-stream' ]) self.assertIn(guess_mime_type('office.ods'), [ 'application/vnd.oasis.opendocument.spreadsheet', 'application/octet-stream' ]) self.assertIn(guess_mime_type('office.odp'), [ 'application/vnd.oasis.opendocument.presentation', 'application/octet-stream' ]) else: # Text self.assertEquals(guess_mime_type('text.csv'), 'text/csv') self.assertEquals(guess_mime_type('text.xml'), 'application/xml') # Image self.assertEquals(guess_mime_type('picture.svg'), 'image/svg+xml') # Video self.assertEquals(guess_mime_type('video.flv'), 'video/x-flv') # Office self.assertEquals( guess_mime_type('office.docx'), 'application/vnd.openxmlformats-officedocument' '.wordprocessingml.document') self.assertEquals( guess_mime_type('office.xlsx'), 'application/vnd.openxmlformats-officedocument' '.spreadsheetml.sheet') self.assertEquals( guess_mime_type('office.pptx'), 'application/vnd.openxmlformats-officedocument' '.presentationml.presentation') self.assertEquals(guess_mime_type('office.odt'), 'application/vnd.oasis.opendocument.text') self.assertEquals( guess_mime_type('office.ods'), 'application/vnd.oasis.opendocument.spreadsheet') self.assertEquals( guess_mime_type('office.odp'), 'application/vnd.oasis.opendocument.presentation')
def test_guess_mime_type(self): # Text self.assertEquals(guess_mime_type('text.txt'), 'text/plain') self.assertEquals(guess_mime_type('text.html'), 'text/html') self.assertEquals(guess_mime_type('text.css'), 'text/css') self.assertEquals(guess_mime_type('text.js'), 'application/javascript') # Image self.assertEquals(guess_mime_type('picture.jpg'), 'image/jpeg') self.assertEquals(guess_mime_type('picture.png'), 'image/png') self.assertEquals(guess_mime_type('picture.gif'), 'image/gif') self.assertIn(guess_mime_type('picture.bmp'), ['image/x-ms-bmp', 'image/bmp']) self.assertEquals(guess_mime_type('picture.tiff'), 'image/tiff') self.assertIn(guess_mime_type('picture.ico'), ['image/x-icon', 'image/vnd.microsoft.icon']) # Audio self.assertEquals(guess_mime_type('sound.mp3'), 'audio/mpeg') self.assertIn(guess_mime_type('sound.wma'), ['audio/x-ms-wma', 'application/octet-stream']) self.assertIn(guess_mime_type('sound.wav'), ['audio/x-wav', 'audio/wav']) # Video self.assertEquals(guess_mime_type('video.mpeg'), 'video/mpeg') self.assertEquals(guess_mime_type('video.mp4'), 'video/mp4') self.assertEquals(guess_mime_type('video.mov'), 'video/quicktime') self.assertIn(guess_mime_type('video.wmv'), ['video/x-ms-wmv', 'application/octet-stream']) self.assertIn(guess_mime_type('video.avi'), ['video/x-msvideo', 'video/avi']) # Office self.assertEquals(guess_mime_type('office.doc'), 'application/msword') self.assertEquals(guess_mime_type('office.xls'), 'application/vnd.ms-excel') self.assertEquals(guess_mime_type('office.ppt'), 'application/vnd.ms-powerpoint') # PDF self.assertEquals(guess_mime_type('document.pdf'), 'application/pdf') # Unknown self.assertEquals(guess_mime_type('file.unknown'), 'application/octet-stream') # Cases badly handled by Windows # See https://jira.nuxeo.com/browse/NXP-11660 # and http://bugs.python.org/issue15207 if sys.platform == "win32": # Text self.assertEquals(guess_mime_type('text.csv'), 'application/octet-stream') self.assertEquals(guess_mime_type('text.xml'), 'text/xml') # Image self.assertIn(guess_mime_type('picture.svg'), ['image/svg+xml', 'application/octet-stream']) # Video self.assertEquals(guess_mime_type('video.flv'), 'application/octet-stream') # Office self.assertIn(guess_mime_type('office.docx'), ['application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/octet-stream']) self.assertIn(guess_mime_type('office.xlsx'), ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', 'application/octet-stream']) self.assertIn(guess_mime_type('office.pptx'), ['application/vnd.openxmlformats-officedocument.presentationml.presentation', 'application/x-mspowerpoint.12', 'application/octet-stream']) self.assertIn(guess_mime_type('office.odt'), ['application/vnd.oasis.opendocument.text', 'application/octet-stream']) self.assertIn(guess_mime_type('office.ods'), ['application/vnd.oasis.opendocument.spreadsheet', 'application/octet-stream']) self.assertIn(guess_mime_type('office.odp'), ['application/vnd.oasis.opendocument.presentation', 'application/octet-stream']) else: # Text self.assertEquals(guess_mime_type('text.csv'), 'text/csv') self.assertEquals(guess_mime_type('text.xml'), 'application/xml') # Image self.assertEquals(guess_mime_type('picture.svg'), 'image/svg+xml') # Video self.assertEquals(guess_mime_type('video.flv'), 'video/x-flv') # Office self.assertEquals(guess_mime_type('office.docx'), 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') self.assertEquals(guess_mime_type('office.xlsx'), 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') self.assertEquals(guess_mime_type('office.pptx'), 'application/vnd.openxmlformats-officedocument.presentationml.presentation') self.assertEquals(guess_mime_type('office.odt'), 'application/vnd.oasis.opendocument.text') self.assertEquals(guess_mime_type('office.ods'), 'application/vnd.oasis.opendocument.spreadsheet') self.assertEquals(guess_mime_type('office.odp'), 'application/vnd.oasis.opendocument.presentation')