def execute_with_blob_streaming(self, command, file_path, filename=None,
mime_type=None, **params):
"""Execute an Automation operation using a batch upload as an input
Upload is streamed.
"""
batch_id = self._generate_unique_id()
tick = time.time()
action = FileAction("Upload", file_path, filename)
try:
upload_result = self.upload(batch_id, file_path, filename=filename,
mime_type=mime_type)
upload_duration = int(time.time() - tick)
action.transfer_duration = upload_duration
# Use upload duration * 2 as Nuxeo transaction timeout
tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2)
log.trace('Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo'
' transaction timeout for batch execution of %s'
' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT,
upload_duration, command, file_path)
if upload_duration > 0:
log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration)
if upload_result['uploaded'] == 'true':
result = self.execute_batch(command, batch_id, '0', tx_timeout,
**params)
return result
else:
raise ValueError("Bad response from batch upload with id '%s'"
" and file path '%s'" % (batch_id, file_path))
finally:
self.end_action()
def test_file_action(tmp):
parent = tmp()
parent.mkdir()
filepath = parent / "test.txt"
size = filepath.write_bytes(b"This is Sparta!")
action = FileAction("Mocking", filepath, size)
assert action.type == "Mocking"
assert not action.empty
# Will test .get_percent()
details = action.export()
assert details["action_type"] == "Mocking"
assert details["progress"] == 0.0
assert isinstance(details["uid"], str)
assert details["size"] == size
assert details["name"] == filepath.name
assert details["filepath"] == str(filepath)
assert Action.get_current_action() is action
# Test repr() when .get_percent() > 0
action.size = 42
action.progress = 4.2
assert repr(action) == "Mocking('test.txt'[42]-10.0)"
Action.finish_action()
assert action.finished
def execute_with_blob_streaming(self,
command,
file_path,
filename=None,
mime_type=None,
**params):
"""Execute an Automation operation using a batch upload as an input
Upload is streamed.
"""
tick = time.time()
action = FileAction("Upload", file_path, filename)
try:
batch_id = None
if self.is_new_upload_api_available():
try:
# Init resumable upload getting a batch id generated by the server
# This batch id is to be used as a resumable session id
batch_id = self.init_upload()['batchId']
except NewUploadAPINotAvailable:
log.debug('New upload API is not available on server %s',
self.server_url)
self.new_upload_api_available = False
if batch_id is None:
# New upload API is not available, generate a batch id
batch_id = self._generate_unique_id()
upload_result = self.upload(batch_id,
file_path,
filename=filename,
mime_type=mime_type)
upload_duration = int(time.time() - tick)
action.transfer_duration = upload_duration
# Use upload duration * 2 as Nuxeo transaction timeout
tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2)
log.trace(
'Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo'
' transaction timeout for batch execution of %s'
' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT,
upload_duration, command, file_path)
if upload_duration > 0:
log.trace("Speed for %d o is %d s : %f o/s",
os.stat(file_path).st_size, upload_duration,
os.stat(file_path).st_size / upload_duration)
# NXDRIVE-433: Compat with 7.4 intermediate state
if upload_result.get('uploaded') is None:
self.new_upload_api_available = False
if upload_result.get('batchId') is not None:
result = self.execute_batch(command, batch_id, '0', tx_timeout,
**params)
return result
else:
raise ValueError("Bad response from batch upload with id '%s'"
" and file path '%s'" % (batch_id, file_path))
except InvalidBatchException:
self.cookie_jar.clear_session_cookies()
finally:
self.end_action()
def test_file_action_with_values():
filepath = Path("fake/test.odt")
action = FileAction("Mocking", filepath, size=42)
assert action.type == "Mocking"
# Will test .get_percent()
details = action.export()
assert details["size"] == 42
assert details["name"] == "test.odt"
assert details["filepath"] == f"fake{os.path.sep}test.odt"
def execute_with_blob_streaming(self, command, file_path, filename=None,
mime_type=None, **params):
"""Execute an Automation operation using a batch upload as an input
Upload is streamed.
"""
tick = time.time()
action = FileAction("Upload", file_path, filename)
retry = True
num_retries = 0
while retry and num_retries < 2:
try:
batch_id = None
if self.is_new_upload_api_available():
try:
# Init resumable upload getting a batch id generated by the server
# This batch id is to be used as a resumable session id
batch_id = self.init_upload()['batchId']
except NewUploadAPINotAvailable:
log.debug('New upload API is not available on server %s', self.server_url)
self.new_upload_api_available = False
if batch_id is None:
# New upload API is not available, generate a batch id
batch_id = self._generate_unique_id()
upload_result = self.upload(batch_id, file_path, filename=filename,
mime_type=mime_type)
upload_duration = int(time.time() - tick)
action.transfer_duration = upload_duration
# Use upload duration * 2 as Nuxeo transaction timeout
tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2)
log.trace('Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo'
' transaction timeout for batch execution of %s'
' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT,
upload_duration, command, file_path)
if upload_duration > 0:
log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration)
# NXDRIVE-433: Compat with 7.4 intermediate state
if upload_result.get('uploaded') is None:
self.new_upload_api_available = False
if upload_result.get('batchId') is not None:
result = self.execute_batch(command, batch_id, '0', tx_timeout,
**params)
return result
else:
raise ValueError("Bad response from batch upload with id '%s'"
" and file path '%s'" % (batch_id, file_path))
except InvalidBatchException:
num_retries += 1
self.cookie_jar.clear_session_cookies()
finally:
self.end_action()
def stream_content(self,
fs_item_id,
file_path,
parent_fs_item_id=None,
fs_item_info=None,
file_out=None):
"""Stream the binary content of a file system item to a tmp file
Raises NotFound if file system item with id fs_item_id
cannot be found
"""
if fs_item_info is None:
fs_item_info = self.get_info(fs_item_id,
parent_fs_item_id=parent_fs_item_id)
download_url = self.server_url + fs_item_info.download_url
file_name = os.path.basename(file_path)
if file_out is None:
file_dir = os.path.dirname(file_path)
file_out = os.path.join(
file_dir, DOWNLOAD_TMP_FILE_PREFIX + file_name +
str(current_thread().ident) + DOWNLOAD_TMP_FILE_SUFFIX)
FileAction("Download", file_out, file_name, 0)
try:
_, tmp_file = self.do_get(
download_url,
file_out=file_out,
digest=fs_item_info.digest,
digest_algorithm=fs_item_info.digest_algorithm)
except Exception as e:
if os.path.exists(file_out):
os.remove(file_out)
raise e
finally:
self.end_action()
return tmp_file
def upload(self, batch_id, file_path, filename=None, file_index=0,
mime_type=None):
"""Upload a file through an Automation batch
Uses poster.httpstreaming to stream the upload
and not load the whole file in memory.
"""
FileAction("Upload", file_path, filename)
# Request URL
if self.is_new_upload_api_available():
url = self.rest_api_url + self.batch_upload_path + '/' + batch_id + '/' + str(file_index)
else:
# Backward compatibility with old batch upload API
url = self.automation_url.encode('ascii') + self.batch_upload_url
# HTTP headers
if filename is None:
filename = os.path.basename(file_path)
file_size = os.path.getsize(file_path)
if mime_type is None:
mime_type = guess_mime_type(filename)
# Quote UTF-8 filenames even though JAX-RS does not seem to be able
# to retrieve them as per: https://tools.ietf.org/html/rfc5987
filename = safe_filename(filename)
quoted_filename = urllib2.quote(filename.encode('utf-8'))
headers = {
"X-File-Name": quoted_filename,
"X-File-Size": file_size,
"X-File-Type": mime_type,
"Content-Type": "application/octet-stream",
"Content-Length": file_size,
}
if not self.is_new_upload_api_available():
headers.update({"X-Batch-Id": batch_id, "X-File-Idx": file_index})
headers.update(self._get_common_headers())
# Request data
input_file = open(file_path, 'rb')
# Use file system block size if available for streaming buffer
fs_block_size = self.get_upload_buffer(input_file)
data = self._read_data(input_file, fs_block_size)
# Execute request
cookies = self._get_cookies()
log.trace("Calling %s with headers %r and cookies %r for file %s",
url, headers, cookies, file_path)
req = urllib2.Request(url, data, headers)
try:
resp = self.streaming_opener.open(req, timeout=self.blob_timeout)
except Exception as e:
log_details = self._log_details(e)
if isinstance(log_details, tuple):
_, _, _, error = log_details
if error and error.startswith("Unable to find batch"):
raise InvalidBatchException()
raise e
finally:
input_file.close()
self.end_action()
return self._read_response(resp, url)
def test_file_action_with_values():
filepath = Path("fake/test.odt")
action = FileAction("Mocking", filepath, 42)
assert action.type == "Mocking"
# Test repr() when .get_percent() equals 0
assert repr(action) == "Mocking('test.odt'[42])"
# Will test .get_percent()
details = action.export()
assert details["size"] == 42
assert details["name"] == "test.odt"
assert details["filepath"] == f"fake{os.path.sep}test.odt"
# Test progress property setter when .progress < .size
action.progress = 24.5
details = action.export()
assert details["progress"] == 24.5 * 100 / 42.0
# Test progress property setter when .progress >= .size
action.progress = 222.0
details = action.export()
assert details["progress"] == 100.0
assert details["uploaded"]
Action.finish_action()
def test_file_action_inexistant_file(tmp):
parent = tmp()
parent.mkdir()
filepath = parent / "test.txt"
action = FileAction("Mocking", filepath, 0)
assert action.empty
assert not action.uploaded
details = action.export()
assert details["action_type"] == "Mocking"
assert details["progress"] == 0.0
assert isinstance(details["uid"], str)
assert details["size"] == 0
assert details["name"] == filepath.name
assert details["filepath"] == str(filepath)
Action.finish_action()
def execute_with_blob_streaming(self, command, file_path, filename=None,
mime_type=None, **params):
"""Execute an Automation operation using a batch upload as an input
Upload is streamed.
"""
tick = time.time()
action = FileAction("Upload", file_path, filename)
try:
batch_id = None
if self.is_chunking_upload_available():
try:
# Init resumable upload getting a batch id generated by the server
# This batch id is to be used as a resumable session id
batch_id = self.init_upload()['batchId']
except Exception:
log.debug('Chunking upload API is not available on server %s', self.server_url)
self.chunking_upload_available = False
if batch_id is None:
# Chunking upload API is not available, generate a batch id
batch_id = self._generate_unique_id()
upload_result = self.upload(batch_id, file_path, filename=filename,
mime_type=mime_type)
upload_duration = int(time.time() - tick)
action.transfer_duration = upload_duration
# Use upload duration * 2 as Nuxeo transaction timeout
tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2)
log.trace('Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo'
' transaction timeout for batch execution of %s'
' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT,
upload_duration, command, file_path)
if upload_duration > 0:
log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration)
if upload_result['uploaded'] == 'true':
result = self.execute_batch(command, batch_id, '0', tx_timeout,
**params)
return result
else:
raise ValueError("Bad response from batch upload with id '%s'"
" and file path '%s'" % (batch_id, file_path))
finally:
self.end_action()
def test_file_action(tmp):
parent = tmp()
parent.mkdir()
filepath = parent / "test.txt"
size = filepath.write_bytes(b"This is Sparta!")
action = FileAction("Mocking", filepath)
assert action.type == "Mocking"
# Will test .get_percent()
details = action.export()
assert details["last_transfer"] == "Mocking"
assert details["progress"] == 0.0
assert isinstance(details["uid"], str)
assert details["size"] == size
assert details["name"] == filepath.name
assert details["filepath"] == str(filepath)
assert Action.get_current_action() == action
Action.finish_action()
assert action.finished
def test_file_action_empty_file(tmp):
parent = tmp()
parent.mkdir()
filepath = parent / "test.txt"
filepath.touch()
action = FileAction("Mocking", filepath, filepath.stat().st_size)
assert action.empty
assert not action.uploaded
details = action.export()
assert details["action_type"] == "Mocking"
assert details["progress"] == 0.0
assert isinstance(details["uid"], str)
assert details["size"] == 0
assert details["name"] == filepath.name
assert details["filepath"] == str(filepath)
# Trigger a progression update telling that the file has been uploaded
action.progress += 0
assert action.export()["progress"] == 100.0
assert action.uploaded
Action.finish_action()
def get_content(self, fs_item_id):
"""Download and return the binary content of a file system item
Beware that the content is loaded in memory.
Raises NotFound if file system item with id fs_item_id
cannot be found
"""
fs_item_info = self.get_info(fs_item_id)
download_url = self.server_url + fs_item_info.download_url
FileAction("Download", None, fs_item_info.name, 0)
content, _ = self.do_get(download_url, digest=fs_item_info.digest,
digest_algorithm=fs_item_info.digest_algorithm)
self.end_action()
return content
def test_file_action_signals():
"""Try to mimic QThread signals to test ._connect_reporter()."""
class Reporter:
def action_started(self):
pass
def action_progressing(self):
pass
def action_done(self):
pass
filepath = Path("fake/test.odt")
action = FileAction("Mocking", filepath, size=42, reporter=Reporter())
Action.finish_action()
assert action.finished
