def execute_with_blob_streaming(self, command, file_path, filename=None, mime_type=None, **params): """Execute an Automation operation using a batch upload as an input Upload is streamed. """ batch_id = self._generate_unique_id() tick = time.time() action = FileAction("Upload", file_path, filename) try: upload_result = self.upload(batch_id, file_path, filename=filename, mime_type=mime_type) upload_duration = int(time.time() - tick) action.transfer_duration = upload_duration # Use upload duration * 2 as Nuxeo transaction timeout tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2) log.trace('Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo' ' transaction timeout for batch execution of %s' ' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT, upload_duration, command, file_path) if upload_duration > 0: log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration) if upload_result['uploaded'] == 'true': result = self.execute_batch(command, batch_id, '0', tx_timeout, **params) return result else: raise ValueError("Bad response from batch upload with id '%s'" " and file path '%s'" % (batch_id, file_path)) finally: self.end_action()
def test_file_action(tmp): parent = tmp() parent.mkdir() filepath = parent / "test.txt" size = filepath.write_bytes(b"This is Sparta!") action = FileAction("Mocking", filepath, size) assert action.type == "Mocking" assert not action.empty # Will test .get_percent() details = action.export() assert details["action_type"] == "Mocking" assert details["progress"] == 0.0 assert isinstance(details["uid"], str) assert details["size"] == size assert details["name"] == filepath.name assert details["filepath"] == str(filepath) assert Action.get_current_action() is action # Test repr() when .get_percent() > 0 action.size = 42 action.progress = 4.2 assert repr(action) == "Mocking('test.txt'[42]-10.0)" Action.finish_action() assert action.finished
def execute_with_blob_streaming(self, command, file_path, filename=None, mime_type=None, **params): """Execute an Automation operation using a batch upload as an input Upload is streamed. """ tick = time.time() action = FileAction("Upload", file_path, filename) try: batch_id = None if self.is_new_upload_api_available(): try: # Init resumable upload getting a batch id generated by the server # This batch id is to be used as a resumable session id batch_id = self.init_upload()['batchId'] except NewUploadAPINotAvailable: log.debug('New upload API is not available on server %s', self.server_url) self.new_upload_api_available = False if batch_id is None: # New upload API is not available, generate a batch id batch_id = self._generate_unique_id() upload_result = self.upload(batch_id, file_path, filename=filename, mime_type=mime_type) upload_duration = int(time.time() - tick) action.transfer_duration = upload_duration # Use upload duration * 2 as Nuxeo transaction timeout tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2) log.trace( 'Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo' ' transaction timeout for batch execution of %s' ' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT, upload_duration, command, file_path) if upload_duration > 0: log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration) # NXDRIVE-433: Compat with 7.4 intermediate state if upload_result.get('uploaded') is None: self.new_upload_api_available = False if upload_result.get('batchId') is not None: result = self.execute_batch(command, batch_id, '0', tx_timeout, **params) return result else: raise ValueError("Bad response from batch upload with id '%s'" " and file path '%s'" % (batch_id, file_path)) except InvalidBatchException: self.cookie_jar.clear_session_cookies() finally: self.end_action()
def test_file_action_with_values(): filepath = Path("fake/test.odt") action = FileAction("Mocking", filepath, size=42) assert action.type == "Mocking" # Will test .get_percent() details = action.export() assert details["size"] == 42 assert details["name"] == "test.odt" assert details["filepath"] == f"fake{os.path.sep}test.odt"
def execute_with_blob_streaming(self, command, file_path, filename=None, mime_type=None, **params): """Execute an Automation operation using a batch upload as an input Upload is streamed. """ tick = time.time() action = FileAction("Upload", file_path, filename) retry = True num_retries = 0 while retry and num_retries < 2: try: batch_id = None if self.is_new_upload_api_available(): try: # Init resumable upload getting a batch id generated by the server # This batch id is to be used as a resumable session id batch_id = self.init_upload()['batchId'] except NewUploadAPINotAvailable: log.debug('New upload API is not available on server %s', self.server_url) self.new_upload_api_available = False if batch_id is None: # New upload API is not available, generate a batch id batch_id = self._generate_unique_id() upload_result = self.upload(batch_id, file_path, filename=filename, mime_type=mime_type) upload_duration = int(time.time() - tick) action.transfer_duration = upload_duration # Use upload duration * 2 as Nuxeo transaction timeout tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2) log.trace('Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo' ' transaction timeout for batch execution of %s' ' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT, upload_duration, command, file_path) if upload_duration > 0: log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration) # NXDRIVE-433: Compat with 7.4 intermediate state if upload_result.get('uploaded') is None: self.new_upload_api_available = False if upload_result.get('batchId') is not None: result = self.execute_batch(command, batch_id, '0', tx_timeout, **params) return result else: raise ValueError("Bad response from batch upload with id '%s'" " and file path '%s'" % (batch_id, file_path)) except InvalidBatchException: num_retries += 1 self.cookie_jar.clear_session_cookies() finally: self.end_action()
def stream_content(self, fs_item_id, file_path, parent_fs_item_id=None, fs_item_info=None, file_out=None): """Stream the binary content of a file system item to a tmp file Raises NotFound if file system item with id fs_item_id cannot be found """ if fs_item_info is None: fs_item_info = self.get_info(fs_item_id, parent_fs_item_id=parent_fs_item_id) download_url = self.server_url + fs_item_info.download_url file_name = os.path.basename(file_path) if file_out is None: file_dir = os.path.dirname(file_path) file_out = os.path.join( file_dir, DOWNLOAD_TMP_FILE_PREFIX + file_name + str(current_thread().ident) + DOWNLOAD_TMP_FILE_SUFFIX) FileAction("Download", file_out, file_name, 0) try: _, tmp_file = self.do_get( download_url, file_out=file_out, digest=fs_item_info.digest, digest_algorithm=fs_item_info.digest_algorithm) except Exception as e: if os.path.exists(file_out): os.remove(file_out) raise e finally: self.end_action() return tmp_file
def upload(self, batch_id, file_path, filename=None, file_index=0, mime_type=None): """Upload a file through an Automation batch Uses poster.httpstreaming to stream the upload and not load the whole file in memory. """ FileAction("Upload", file_path, filename) # Request URL if self.is_new_upload_api_available(): url = self.rest_api_url + self.batch_upload_path + '/' + batch_id + '/' + str(file_index) else: # Backward compatibility with old batch upload API url = self.automation_url.encode('ascii') + self.batch_upload_url # HTTP headers if filename is None: filename = os.path.basename(file_path) file_size = os.path.getsize(file_path) if mime_type is None: mime_type = guess_mime_type(filename) # Quote UTF-8 filenames even though JAX-RS does not seem to be able # to retrieve them as per: https://tools.ietf.org/html/rfc5987 filename = safe_filename(filename) quoted_filename = urllib2.quote(filename.encode('utf-8')) headers = { "X-File-Name": quoted_filename, "X-File-Size": file_size, "X-File-Type": mime_type, "Content-Type": "application/octet-stream", "Content-Length": file_size, } if not self.is_new_upload_api_available(): headers.update({"X-Batch-Id": batch_id, "X-File-Idx": file_index}) headers.update(self._get_common_headers()) # Request data input_file = open(file_path, 'rb') # Use file system block size if available for streaming buffer fs_block_size = self.get_upload_buffer(input_file) data = self._read_data(input_file, fs_block_size) # Execute request cookies = self._get_cookies() log.trace("Calling %s with headers %r and cookies %r for file %s", url, headers, cookies, file_path) req = urllib2.Request(url, data, headers) try: resp = self.streaming_opener.open(req, timeout=self.blob_timeout) except Exception as e: log_details = self._log_details(e) if isinstance(log_details, tuple): _, _, _, error = log_details if error and error.startswith("Unable to find batch"): raise InvalidBatchException() raise e finally: input_file.close() self.end_action() return self._read_response(resp, url)
def test_file_action_with_values(): filepath = Path("fake/test.odt") action = FileAction("Mocking", filepath, 42) assert action.type == "Mocking" # Test repr() when .get_percent() equals 0 assert repr(action) == "Mocking('test.odt'[42])" # Will test .get_percent() details = action.export() assert details["size"] == 42 assert details["name"] == "test.odt" assert details["filepath"] == f"fake{os.path.sep}test.odt" # Test progress property setter when .progress < .size action.progress = 24.5 details = action.export() assert details["progress"] == 24.5 * 100 / 42.0 # Test progress property setter when .progress >= .size action.progress = 222.0 details = action.export() assert details["progress"] == 100.0 assert details["uploaded"] Action.finish_action()
def test_file_action_inexistant_file(tmp): parent = tmp() parent.mkdir() filepath = parent / "test.txt" action = FileAction("Mocking", filepath, 0) assert action.empty assert not action.uploaded details = action.export() assert details["action_type"] == "Mocking" assert details["progress"] == 0.0 assert isinstance(details["uid"], str) assert details["size"] == 0 assert details["name"] == filepath.name assert details["filepath"] == str(filepath) Action.finish_action()
def execute_with_blob_streaming(self, command, file_path, filename=None, mime_type=None, **params): """Execute an Automation operation using a batch upload as an input Upload is streamed. """ tick = time.time() action = FileAction("Upload", file_path, filename) try: batch_id = None if self.is_chunking_upload_available(): try: # Init resumable upload getting a batch id generated by the server # This batch id is to be used as a resumable session id batch_id = self.init_upload()['batchId'] except Exception: log.debug('Chunking upload API is not available on server %s', self.server_url) self.chunking_upload_available = False if batch_id is None: # Chunking upload API is not available, generate a batch id batch_id = self._generate_unique_id() upload_result = self.upload(batch_id, file_path, filename=filename, mime_type=mime_type) upload_duration = int(time.time() - tick) action.transfer_duration = upload_duration # Use upload duration * 2 as Nuxeo transaction timeout tx_timeout = max(DEFAULT_NUXEO_TX_TIMEOUT, upload_duration * 2) log.trace('Using %d seconds [max(%d, 2 * upload time=%d)] as Nuxeo' ' transaction timeout for batch execution of %s' ' with file %s', tx_timeout, DEFAULT_NUXEO_TX_TIMEOUT, upload_duration, command, file_path) if upload_duration > 0: log.trace("Speed for %d o is %d s : %f o/s", os.stat(file_path).st_size, upload_duration, os.stat(file_path).st_size / upload_duration) if upload_result['uploaded'] == 'true': result = self.execute_batch(command, batch_id, '0', tx_timeout, **params) return result else: raise ValueError("Bad response from batch upload with id '%s'" " and file path '%s'" % (batch_id, file_path)) finally: self.end_action()
def test_file_action(tmp): parent = tmp() parent.mkdir() filepath = parent / "test.txt" size = filepath.write_bytes(b"This is Sparta!") action = FileAction("Mocking", filepath) assert action.type == "Mocking" # Will test .get_percent() details = action.export() assert details["last_transfer"] == "Mocking" assert details["progress"] == 0.0 assert isinstance(details["uid"], str) assert details["size"] == size assert details["name"] == filepath.name assert details["filepath"] == str(filepath) assert Action.get_current_action() == action Action.finish_action() assert action.finished
def test_file_action_empty_file(tmp): parent = tmp() parent.mkdir() filepath = parent / "test.txt" filepath.touch() action = FileAction("Mocking", filepath, filepath.stat().st_size) assert action.empty assert not action.uploaded details = action.export() assert details["action_type"] == "Mocking" assert details["progress"] == 0.0 assert isinstance(details["uid"], str) assert details["size"] == 0 assert details["name"] == filepath.name assert details["filepath"] == str(filepath) # Trigger a progression update telling that the file has been uploaded action.progress += 0 assert action.export()["progress"] == 100.0 assert action.uploaded Action.finish_action()
def get_content(self, fs_item_id): """Download and return the binary content of a file system item Beware that the content is loaded in memory. Raises NotFound if file system item with id fs_item_id cannot be found """ fs_item_info = self.get_info(fs_item_id) download_url = self.server_url + fs_item_info.download_url FileAction("Download", None, fs_item_info.name, 0) content, _ = self.do_get(download_url, digest=fs_item_info.digest, digest_algorithm=fs_item_info.digest_algorithm) self.end_action() return content
def test_file_action_signals(): """Try to mimic QThread signals to test ._connect_reporter().""" class Reporter: def action_started(self): pass def action_progressing(self): pass def action_done(self): pass filepath = Path("fake/test.odt") action = FileAction("Mocking", filepath, size=42, reporter=Reporter()) Action.finish_action() assert action.finished