def upload(self, batch_id, file_path, filename=None, file_index=0,
mime_type=None):
"""Upload a file through an Automation batch
Uses poster.httpstreaming to stream the upload
and not load the whole file in memory.
"""
FileAction("Upload", file_path, filename)
# Request URL
if self.is_new_upload_api_available():
url = self.rest_api_url + self.batch_upload_path + '/' + batch_id + '/' + str(file_index)
else:
# Backward compatibility with old batch upload API
url = self.automation_url.encode('ascii') + self.batch_upload_url
# HTTP headers
if filename is None:
filename = os.path.basename(file_path)
file_size = os.path.getsize(file_path)
if mime_type is None:
mime_type = guess_mime_type(filename)
# Quote UTF-8 filenames even though JAX-RS does not seem to be able
# to retrieve them as per: https://tools.ietf.org/html/rfc5987
filename = safe_filename(filename)
quoted_filename = urllib2.quote(filename.encode('utf-8'))
headers = {
"X-File-Name": quoted_filename,
"X-File-Size": file_size,
"X-File-Type": mime_type,
"Content-Type": "application/octet-stream",
"Content-Length": file_size,
}
if not self.is_new_upload_api_available():
headers.update({"X-Batch-Id": batch_id, "X-File-Idx": file_index})
headers.update(self._get_common_headers())
# Request data
input_file = open(file_path, 'rb')
# Use file system block size if available for streaming buffer
fs_block_size = self.get_upload_buffer(input_file)
data = self._read_data(input_file, fs_block_size)
# Execute request
cookies = self._get_cookies()
log.trace("Calling %s with headers %r and cookies %r for file %s",
url, headers, cookies, file_path)
req = urllib2.Request(url, data, headers)
try:
resp = self.streaming_opener.open(req, timeout=self.blob_timeout)
except Exception as e:
log_details = self._log_details(e)
if isinstance(log_details, tuple):
_, _, _, error = log_details
if error and error.startswith("Unable to find batch"):
raise InvalidBatchException()
raise e
finally:
input_file.close()
self.end_action()
return self._read_response(resp, url)
def upload(self,
batch_id,
file_path,
filename=None,
file_index=0,
mime_type=None):
"""Upload a file through an Automation batch
Uses poster.httpstreaming to stream the upload
and not load the whole file in memory.
"""
self.current_action = FileAction("Upload", file_path, filename)
# Request URL
url = self.automation_url.encode('ascii') + self.batch_upload_url
# HTTP headers
if filename is None:
filename = os.path.basename(file_path)
file_size = os.path.getsize(file_path)
if mime_type is None:
mime_type = guess_mime_type(filename)
# Quote UTF-8 filenames even though JAX-RS does not seem to be able
# to retrieve them as per: https://tools.ietf.org/html/rfc5987
filename = safe_filename(filename)
quoted_filename = urllib2.quote(filename.encode('utf-8'))
headers = {
"X-Batch-Id": batch_id,
"X-File-Idx": file_index,
"X-File-Name": quoted_filename,
"X-File-Size": file_size,
"X-File-Type": mime_type,
"Content-Type": "application/octet-stream",
"Content-Length": file_size,
}
headers.update(self._get_common_headers())
# Request data
input_file = open(file_path, 'rb')
# Use file system block size if available for streaming buffer
fs_block_size = self.get_upload_buffer(input_file)
log.trace(
"Using file system block size"
" for the streaming upload buffer: %u bytes", fs_block_size)
data = self._read_data(input_file, fs_block_size)
# Execute request
cookies = self._get_cookies()
log.trace("Calling %s with headers %r and cookies %r for file %s", url,
headers, cookies, file_path)
req = urllib2.Request(url, data, headers)
try:
resp = self.streaming_opener.open(req, timeout=self.blob_timeout)
except Exception as e:
self._log_details(e)
raise
finally:
input_file.close()
self.end_action()
return self._read_response(resp, url)
def upload(self, batch_id, file_path, filename=None, file_index=0,
mime_type=None):
"""Upload a file through an Automation batch
Uses poster.httpstreaming to stream the upload
and not load the whole file in memory.
"""
# Request URL
url = self.automation_url.encode('ascii') + self.batch_upload_url
# HTTP headers
if filename is None:
filename = os.path.basename(file_path)
file_size = os.path.getsize(file_path)
if mime_type is None:
mime_type = guess_mime_type(filename)
# Quote UTF-8 filenames even though JAX-RS does not seem to be able
# to retrieve them as per: https://tools.ietf.org/html/rfc5987
filename = safe_filename(filename)
quoted_filename = urllib2.quote(filename.encode('utf-8'))
headers = {
"X-Batch-Id": batch_id,
"X-File-Idx": file_index,
"X-File-Name": quoted_filename,
"X-File-Size": file_size,
"X-File-Type": mime_type,
"Content-Type": "application/octet-stream",
"Content-Length": file_size,
}
headers.update(self._get_common_headers())
# Request data
input_file = open(file_path, 'rb')
# Use file system block size if available for streaming buffer
if sys.platform != 'win32':
fs_block_size = os.fstatvfs(input_file.fileno()).f_bsize
else:
fs_block_size = FILE_BUFFER_SIZE
log.trace("Using file system block size"
" for the streaming upload buffer: %u bytes", fs_block_size)
data = self._read_data(input_file, fs_block_size)
# Execute request
cookies = self._get_cookies()
log.trace("Calling %s with headers %r and cookies %r for file %s",
url, headers, cookies, file_path)
req = urllib2.Request(url, data, headers)
try:
resp = self.streaming_opener.open(req, timeout=self.blob_timeout)
except Exception as e:
self._log_details(e)
raise
finally:
input_file.close()
return self._read_response(resp, url)
def test_guess_mime_type(self):
# Text
self.assertEquals(guess_mime_type('text.txt'), 'text/plain')
self.assertEquals(guess_mime_type('text.html'), 'text/html')
self.assertEquals(guess_mime_type('text.css'), 'text/css')
self.assertEquals(guess_mime_type('text.js'), 'application/javascript')
# Image
self.assertEquals(guess_mime_type('picture.jpg'), 'image/jpeg')
self.assertEquals(guess_mime_type('picture.png'), 'image/png')
self.assertEquals(guess_mime_type('picture.gif'), 'image/gif')
self.assertIn(guess_mime_type('picture.bmp'),
['image/x-ms-bmp', 'image/bmp'])
self.assertEquals(guess_mime_type('picture.tiff'), 'image/tiff')
self.assertIn(guess_mime_type('picture.ico'),
['image/x-icon', 'image/vnd.microsoft.icon'])
# Audio
self.assertEquals(guess_mime_type('sound.mp3'), 'audio/mpeg')
self.assertIn(guess_mime_type('sound.wma'),
['audio/x-ms-wma', 'application/octet-stream'])
self.assertEquals(guess_mime_type('sound.wav'), 'audio/x-wav')
# Video
self.assertEquals(guess_mime_type('video.mpeg'), 'video/mpeg')
self.assertEquals(guess_mime_type('video.mp4'), 'video/mp4')
self.assertEquals(guess_mime_type('video.mov'), 'video/quicktime')
self.assertIn(guess_mime_type('video.wmv'),
['video/x-ms-wmv', 'application/octet-stream'])
self.assertIn(guess_mime_type('video.avi'),
['video/x-msvideo', 'video/avi'])
# Office
self.assertEquals(guess_mime_type('office.doc'), 'application/msword')
self.assertIn(guess_mime_type('office.xls'),
['application/vnd.ms-excel', 'application/x-msexcel'])
self.assertIn(
guess_mime_type('office.ppt'),
['application/vnd.ms-powerpoint', 'application/x-mspowerpoint'])
# PDF
self.assertEquals(guess_mime_type('document.pdf'), 'application/pdf')
# Unknown
self.assertEquals(guess_mime_type('file.unknown'),
'application/octet-stream')
# Cases badly handled by Windows
# See https://jira.nuxeo.com/browse/NXP-11660
# and http://bugs.python.org/issue15207
if sys.platform == "win32":
# Text
self.assertEquals(guess_mime_type('text.csv'),
'application/octet-stream')
self.assertEquals(guess_mime_type('text.xml'), 'text/xml')
# Image
self.assertIn(guess_mime_type('picture.svg'),
['image/svg+xml', 'application/octet-stream'])
# Video
self.assertEquals(guess_mime_type('video.flv'),
'application/octet-stream')
# Office
self.assertIn(guess_mime_type('office.docx'), [
'application/vnd.openxmlformats-officedocument'
'.wordprocessingml.document', 'application/octet-stream'
])
self.assertIn(guess_mime_type('office.xlsx'), [
'application/vnd.openxmlformats-officedocument'
'.spreadsheetml.sheet', 'application/octet-stream'
])
self.assertIn(guess_mime_type('office.pptx'), [
'application/vnd.openxmlformats-officedocument'
'.presentationml.presentation',
'application/x-mspowerpoint.12', 'application/octet-stream'
])
self.assertIn(guess_mime_type('office.odt'), [
'application/vnd.oasis.opendocument.text',
'application/octet-stream'
])
self.assertIn(guess_mime_type('office.ods'), [
'application/vnd.oasis.opendocument.spreadsheet',
'application/octet-stream'
])
self.assertIn(guess_mime_type('office.odp'), [
'application/vnd.oasis.opendocument.presentation',
'application/octet-stream'
])
else:
# Text
self.assertEquals(guess_mime_type('text.csv'), 'text/csv')
self.assertEquals(guess_mime_type('text.xml'), 'application/xml')
# Image
self.assertEquals(guess_mime_type('picture.svg'), 'image/svg+xml')
# Video
self.assertEquals(guess_mime_type('video.flv'), 'video/x-flv')
# Office
self.assertEquals(
guess_mime_type('office.docx'),
'application/vnd.openxmlformats-officedocument'
'.wordprocessingml.document')
self.assertEquals(
guess_mime_type('office.xlsx'),
'application/vnd.openxmlformats-officedocument'
'.spreadsheetml.sheet')
self.assertEquals(
guess_mime_type('office.pptx'),
'application/vnd.openxmlformats-officedocument'
'.presentationml.presentation')
self.assertEquals(guess_mime_type('office.odt'),
'application/vnd.oasis.opendocument.text')
self.assertEquals(
guess_mime_type('office.ods'),
'application/vnd.oasis.opendocument.spreadsheet')
self.assertEquals(
guess_mime_type('office.odp'),
'application/vnd.oasis.opendocument.presentation')
def test_guess_mime_type(self):
# Text
self.assertEquals(guess_mime_type('text.txt'), 'text/plain')
self.assertEquals(guess_mime_type('text.html'), 'text/html')
self.assertEquals(guess_mime_type('text.css'), 'text/css')
self.assertEquals(guess_mime_type('text.js'), 'application/javascript')
# Image
self.assertEquals(guess_mime_type('picture.jpg'), 'image/jpeg')
self.assertEquals(guess_mime_type('picture.png'), 'image/png')
self.assertEquals(guess_mime_type('picture.gif'), 'image/gif')
self.assertIn(guess_mime_type('picture.bmp'), ['image/x-ms-bmp',
'image/bmp'])
self.assertEquals(guess_mime_type('picture.tiff'), 'image/tiff')
self.assertIn(guess_mime_type('picture.ico'), ['image/x-icon', 'image/vnd.microsoft.icon'])
# Audio
self.assertEquals(guess_mime_type('sound.mp3'), 'audio/mpeg')
self.assertIn(guess_mime_type('sound.wma'), ['audio/x-ms-wma', 'application/octet-stream'])
self.assertIn(guess_mime_type('sound.wav'), ['audio/x-wav', 'audio/wav'])
# Video
self.assertEquals(guess_mime_type('video.mpeg'), 'video/mpeg')
self.assertEquals(guess_mime_type('video.mp4'), 'video/mp4')
self.assertEquals(guess_mime_type('video.mov'), 'video/quicktime')
self.assertIn(guess_mime_type('video.wmv'), ['video/x-ms-wmv', 'application/octet-stream'])
self.assertIn(guess_mime_type('video.avi'), ['video/x-msvideo',
'video/avi'])
# Office
self.assertEquals(guess_mime_type('office.doc'),
'application/msword')
self.assertEquals(guess_mime_type('office.xls'),
'application/vnd.ms-excel')
self.assertEquals(guess_mime_type('office.ppt'),
'application/vnd.ms-powerpoint')
# PDF
self.assertEquals(guess_mime_type('document.pdf'),
'application/pdf')
# Unknown
self.assertEquals(guess_mime_type('file.unknown'),
'application/octet-stream')
# Cases badly handled by Windows
# See https://jira.nuxeo.com/browse/NXP-11660
# and http://bugs.python.org/issue15207
if sys.platform == "win32":
# Text
self.assertEquals(guess_mime_type('text.csv'),
'application/octet-stream')
self.assertEquals(guess_mime_type('text.xml'), 'text/xml')
# Image
self.assertIn(guess_mime_type('picture.svg'), ['image/svg+xml', 'application/octet-stream'])
# Video
self.assertEquals(guess_mime_type('video.flv'),
'application/octet-stream')
# Office
self.assertIn(guess_mime_type('office.docx'),
['application/vnd.openxmlformats-officedocument.wordprocessingml.document',
'application/octet-stream'])
self.assertIn(guess_mime_type('office.xlsx'),
['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
'application/octet-stream'])
self.assertIn(guess_mime_type('office.pptx'),
['application/vnd.openxmlformats-officedocument.presentationml.presentation',
'application/x-mspowerpoint.12', 'application/octet-stream'])
self.assertIn(guess_mime_type('office.odt'),
['application/vnd.oasis.opendocument.text', 'application/octet-stream'])
self.assertIn(guess_mime_type('office.ods'),
['application/vnd.oasis.opendocument.spreadsheet', 'application/octet-stream'])
self.assertIn(guess_mime_type('office.odp'),
['application/vnd.oasis.opendocument.presentation', 'application/octet-stream'])
else:
# Text
self.assertEquals(guess_mime_type('text.csv'), 'text/csv')
self.assertEquals(guess_mime_type('text.xml'), 'application/xml')
# Image
self.assertEquals(guess_mime_type('picture.svg'), 'image/svg+xml')
# Video
self.assertEquals(guess_mime_type('video.flv'), 'video/x-flv')
# Office
self.assertEquals(guess_mime_type('office.docx'),
'application/vnd.openxmlformats-officedocument.wordprocessingml.document')
self.assertEquals(guess_mime_type('office.xlsx'),
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')
self.assertEquals(guess_mime_type('office.pptx'),
'application/vnd.openxmlformats-officedocument.presentationml.presentation')
self.assertEquals(guess_mime_type('office.odt'), 'application/vnd.oasis.opendocument.text')
self.assertEquals(guess_mime_type('office.ods'), 'application/vnd.oasis.opendocument.spreadsheet')
self.assertEquals(guess_mime_type('office.odp'), 'application/vnd.oasis.opendocument.presentation')
