def test_process_dies_with_object(self): """ SubProcess for a process that will never stop """ # Set up our threaded process to sleep for 1 day sp = SubProcess(['sleep', '1d']) # Now run our process sp.start() # Wait half a second (give thread time to start and run) sp.join(timeout=0.5) # We'll get the pid number of our process assert sp.pid() is not None pid = sp.pid() # Test that the process is indeed running assert self.pid_exists(pid) is True # now we'll flat out destroy the object (always abort it first) sp.abort() del sp # Test that the process is now dead as a result of the previous action assert self.pid_exists(pid) is False
def test_program_timeout(self): """ SubProcess for a process that will be killed simply because it took too long to execute. """ # Set up our threaded process to sleep for 1 day # Bu sp = SubProcess(['sleep', '1d'], timeout=1.0) # Now run our process sp.start() # Wait until it expires assert sp.is_complete(timeout=5.0) is True # PID is always None unless the process is still running assert sp.pid() is None # The process is not successful, but we shouldn't be relying on this # value anyway because it's not even complete. assert sp.successful() is False # time should be elapsing assert sp.elapsed() > 0.0 # No output to report at this time because one of the downsides of # using subprocess is you can't retrieve the pipes content until the # end. Thus these functions will simply return nothing. assert len(sp.stdout()) > 0 assert len(sp.stderr()) > 0 # A Timeout assert sp.response_code() is ReturnCode.Timeout
def test_simple_list(self): """ SubProcess for listing files """ # Set up our threaded process sp = SubProcess(['ls', '-1', '/tmp']) # Now run our process sp.start() # Wait for it to complete sp.join() assert sp.is_complete() is True assert sp.response_code() is 0 assert sp.successful() is True assert sp.elapsed() > 0.0 assert isinstance(sp.stdout(), list) assert isinstance(sp.stderr(), list) assert isinstance(sp.stdout(as_list=False), basestring) assert isinstance(sp.stderr(as_list=False), basestring) # Because under the hood stdout and stderr are stored into # Stream objects, we need to be able to check that multple # calls to the same object still return the same results assert len(sp.stdout()) > 1 assert len(sp.stdout()) > 1 assert len(sp.stderr()) > 0 assert len(sp.stderr()) > 0
def test_program_that_will_not_die(self): """ SubProcess for a process that will never stop """ # Set up our threaded process to sleep for 1 day sp = SubProcess(['sleep', '1d']) # Now run our process sp.start() # Wait half a second (give thread time to start and run) sp.join(timeout=0.5) # PID is always None unless the process is still running assert sp.pid() is not None assert isinstance(sp.pid(), int) # The process is still running assert sp.is_complete() is False # The process is not successful, but we shouldn't be relying on this # value anyway because it's not even complete. assert sp.successful() is False # time should be elapsing assert sp.elapsed() > 0.0 # No output to report at this time because one of the downsides of # using subprocess is you can't retrieve the pipes content until the # end. Thus these functions will simply return nothing. assert len(sp.stdout()) > 0 assert len(sp.stderr()) > 0 # Currently the response code is still unknown assert sp.response_code() is ReturnCode.Unknown # We'll abort the process now sp.abort() # Wait for it to complete (because it will complete now) sp.join() # PID is always None when process is stopped assert sp.pid() is None # The response code should have change to Aborted assert sp.response_code() is ReturnCode.Aborted # The process is now complete assert sp.is_complete() is True # But we should not be successful at this point assert sp.successful() is False
def test_bad_execution(self): """ SubProcess for listing files with bad arguments """ # Set up our threaded process sp = SubProcess(['ls', '-wtf', '-not-a-valid-argument']) # Now run our process sp.start() # Wait for it to complete sp.join() assert sp.is_complete() is True assert sp.response_code() is not 0 assert sp.successful() is False assert sp.elapsed() > 0.0 # Because under the hood stdout and stderr are stored into # Stream objects, we need to be able to check that multple # calls to the same object still return the same results assert len(sp.stdout()) > 0 assert len(sp.stdout()) > 0 assert len(sp.stderr()) > 1 assert len(sp.stderr()) > 1
def encode(self, content=None, *args, **kwargs): """ Takes a specified path (and or file) and creates par2 files based on it. If this function is successful, it returns a set of NNTPBinaryContent() objects identifying the PAR2 files generated based on the passed in content. The function returns None if it fails in any way. """ if content is not None: self.add(content) # Some simple error checking to save from doing to much here if len(self) == 0: return None if not self.can_exe(self._par): return None for target in self.archive: # Base entry on first file in the list name = basename(target) target_dir = dirname(target) # tmp_path, tmp_file = self.mkstemp(content=name, suffix='.par2') # Initialize our command execute = [ # Our Executable PAR Application self._par, # Use Create Flag 'create', ] # Handle PAR Block Size if self.block_size: execute.append('-s%s' % self.block_size) if self.recovery_percent: execute.append('-r%d' % self.recovery_percent) if self.cpu_cores is not None and self.cpu_cores > 1: # to repair concurrently - uses multiple threads execute.append('-t+') # Stop Switch Parsing execute.append('--') # Now add our target (we can only do one at a time which i why we # loop) and run our setups execute.append(target) found_set = sortedset() with pushd(target_dir): # Create our SubProcess Instance sp = SubProcess(execute) # Start our execution now sp.start() while not sp.is_complete(timeout=1.5): found_set = self.watch_dir( target_dir, prefix=name, regex=PAR_PART_RE, ignore=found_set, ) # Handle remaining content found_set = self.watch_dir( target_dir, prefix=name, regex=PAR_PART_RE, ignore=found_set, seconds=-1, ) # Let the caller know our status if not sp.successful(): # We're done; we failed return None if not len(found_set): # We're done; we failed return None # Create a resultset results = sortedset(key=lambda x: x.key()) part = 0 # iterate through our found_set and create NNTPBinaryContent() # objects from them. for path in found_set: # Iterate over our found files and determine their part # information part += 1 content = NNTPBinaryContent( path, part=part, total_parts=len(found_set), ) # Loaded data is by default detached; we want to attach it content.attach() # Add our attached content to our results results.add(content) # Clean our are list of objects to archive self.clear() # Return our return results
def test(self, content=None): """ content must be pointing to a directory containing par files that can be easily sorted on. Alternatively, path can be of type NNTPContent() or a set/list of. This function just tests an archive to see if it can be properly prepared (it is effectively a wrapper to verify) If anything but True is returned then there was a problem verifying the results and a code identified in ParReturnCode() is returned instead. """ if content is not None: paths = self.get_paths(content) elif len(self.archive): # Get first item in archive paths = iter(self.archive) else: raise AttributeError("CodecPar: No par file detected.") if not self.can_exe(self._par): return None # filter our results by indexes indexes = self.__filter_pars(paths, indexes=True, volumes=False) if not len(indexes): logger.warning('Archive contained no PAR files.') return ParReturnCode.NoParFiles # Initialize our command execute = [ # Our Executable PAR Application self._par, # Use Test Flag 'verify', ] if self.cpu_cores is not None and self.cpu_cores > 1: # to checksum concurrently - uses multiple threads execute.append('-t+') # Stop Switch Parsing execute.append('--') for _path in indexes: # Get the directory the par file resides in par_path = dirname(_path) with pushd(par_path): # Create our SubProcess Instance sp = SubProcess(list(execute) + [basename(_path)]) # Start our execution now sp.start() sp.join() # Let the caller know our status if sp.response_code() is not ParReturnCode.NoRepairRequired: return sp.response_code() return True
def encode(self, content=None, name=None, *args, **kwargs): """ Takes a specified path (and or file) and compresses it. If this function is successful, it returns a set of NNTPBinaryContent() objects that are 'not' detached. The function returns None if it fails in any way """ if content is not None: self.add(content) # Some simple error checking to save from doing to much here if len(self) == 0: return None if not self.can_exe(self._bin): return None if not name: name = self.name if not name: name = random_str() tmp_path, tmp_file = self.mkstemp(content=name, suffix='.7z') # Initialize our command execute = [ # Our Executable 7-Zip Application self._bin, # Use Add Flag 'a', # Default mode is 7-Zip '-t7z', ] # Password Protection if self.password is not None: execute.append('-p%s' % self.password) # Handle Compression Level if self.level is CompressionLevel.Maximum: execute.append('-mx9') elif self.level is CompressionLevel.Average: execute.append('-mx5') elif self.level is CompressionLevel.Minimum: execute.append('-mx1') # Don't prompt for anything execute.append('-y') if not name: name = splitext(basename(tmp_file))[0] # Handle 7Z Volume Splitting if self.volume_size: execute.append('-v%sb' % self.volume_size) if self.cpu_cores is not None and self.cpu_cores > 1: # create archive using multiple threads execute.append('-mmt%d' % self.cpu_cores) # Stop Switch Parsing execute.append('--') # Specify the Destination Path execute.append(tmp_file) # Add all of our paths now for _path in self: execute.append(_path) # Create our SubProcess Instance sp = SubProcess(execute) # Start our execution now sp.start() found_set = None while not sp.is_complete(timeout=1.5): found_set = self.watch_dir( tmp_path, prefix=name, ignore=found_set, ) # Handle remaining content found_set = self.watch_dir( tmp_path, prefix=name, ignore=found_set, seconds=-1, ) # Let the caller know our status if not sp.successful(): # Cleanup Temporary Path rm(tmp_path) return None if not len(found_set): return None # Create a resultset results = sortedset(key=lambda x: x.key()) # iterate through our found_set and create NNTPBinaryContent() # objects from them. part = 0 for path in found_set: # Iterate over our found files and determine their part # information _re_results = SEVEN_ZIP_PART_RE.match(path) if _re_results: if _re_results.group('part') is not None: part = int(_re_results.group('part')) elif _re_results.group('part0') is not None: part = int(_re_results.group('part0')) else: part += 1 else: part += 1 content = NNTPBinaryContent( path, part=part, total_parts=len(found_set), ) # Loaded data is by default detached; we want to attach it content.attach() # Add our attached content to our results results.add(content) # Clean our are list of objects to archive self.clear() # Return our return results
def decode(self, content=None, name=None, password=None, *args, **kwargs): """ content must be pointing to a directory containing 7-Zip files that can be easily sorted on. Alternatively, path can be of type NNTPContent() or a set/list of. If no password is specified, then the password configuration loaded into the class is used instead. An NNTPBinaryContent() object containing the contents of the package within a sortedset() object. All decoded() functions have to return a resultset() to be consistent with one another. """ if content is not None: self.add(content) # Some simple error checking to save from doing to much here if len(self) == 0: return None if not self.can_exe(self._bin): return None if not password: password = self.password # Initialize our command execute = [ # Our Executable 7-Zip Application self._bin, # Use Add Flag 'x', # Assume Yes '-y', ] # Password Protection if password is not None: execute.append('-p%s' % password) else: # Do not prompt for password execute.append('-p-') if self.overwrite: # Overwrite files execute.append('-aoa') else: # Don't overwrite files execute.append('-aos') # Stop Switch Parsing execute.append('--') if not name: name = self.name if not name: name = random_str() for _path in self: # Temporary Path tmp_path, _ = self.mkstemp(content=name) with pushd(tmp_path): # Create our SubProcess Instance sp = SubProcess(list(execute) + [_path]) # Start our execution now sp.start() found_set = None while not sp.is_complete(timeout=1.5): found_set = self.watch_dir( tmp_path, ignore=found_set, ) # Handle remaining content found_set = self.watch_dir( tmp_path, ignore=found_set, seconds=-1, ) # Let the caller know our status if not sp.successful(): # Cleanup Temporary Path rm(tmp_path) return None if not len(found_set): logger.warning( '7Z archive (%s) contained no content.' % basename(_path), ) # Clean our are list of objects to archive self.clear() # Return path containing unrar'ed content results = NNTPBinaryContent(tmp_path) # We intentionally attach it's content results.attach() # Create a sortedset to return _resultset = sortedset(key=lambda x: x.key()) _resultset.add(results) # Return our content return _resultset
def test(self, content=None, password=None): """ content must be pointing to a directory containing rar files that can be easily sorted on. Alternatively, path can be of type NNTPContent() or a set/list of. If no password is specified, then the password configuration loaded into the class is used instead. This function just tests an archive to see if it can be properly extracted using the known password. """ if content is not None: paths = self.get_paths(content) elif len(self.archive): # Get first item in archive paths = iter(self.archive) else: raise AttributeError("Codec7Zip: No 7-Zip file detected.") if not self.can_exe(self._bin): return None if not password: password = self.password # Initialize our command execute = [ # Our Executable 7-Zip Application self._bin, # Use Test Flag 't', # Assume Yes '-y', ] # Password Protection if password is not None: execute.append('-p%s' % password) else: # Do not prompt for password execute.append('-p-') # Stop Switch Parsing execute.append('--') for _path in paths: # Create our SubProcess Instance sp = SubProcess(list(execute) + [_path]) # Start our execution now sp.start() sp.join() # Let the caller know our status if not sp.successful(): return False return True
def encode(self, content=None, name=None, *args, **kwargs): """ Takes a specified path (and or file) and compresses it. If this function is successful, it returns a set of NNTPBinaryContent() objects that are 'not' detached. The function returns None if it fails in any way """ if content is not None: self.add(content) # Some simple error checking to save from doing to much here if len(self) == 0: return None if not self.can_exe(self._rar): return None if not name: name = self.name if not name: name = random_str() tmp_path, tmp_file = self.mkstemp(content=name, suffix='.rar') # Initialize our command execute = [ # Our Executable RAR Application self._rar, # Use Add Flag 'a', ] # Password Protection if self.password is not None: execute.append('-p%s' % self.password) # Handle Compression Level if self.level is CompressionLevel.Maximum: execute.append('-m5') elif self.level is CompressionLevel.Average: execute.append('-m3') elif self.level is CompressionLevel.Minimum: execute.append('-m0') # Exclude base directory from archive execute.append('-ep1') if not name: name = splitext(basename(tmp_file))[0] # Now place content within directory identifed by it's name execute.append('-ap%s' % name) # Handle RAR Volume Splitting if self.volume_size: execute.append('-v%sb' % self.volume_size) # Handle Recovery Record if self.recovery_record is not None: execute.append('-rr%s' % self.recovery_record) if self.cpu_cores is not None and self.cpu_cores > 1: # create archive using multiple threads execute.append('-mt%d' % self.cpu_cores) # Stop Switch Parsing execute.append('--') # Specify the Destination Path execute.append(tmp_file) # Add all of our paths now for _path in self: execute.append(_path) # Create our SubProcess Instance sp = SubProcess(execute) # Start our execution now sp.start() found_set = None while not sp.is_complete(timeout=1.5): found_set = self.watch_dir( tmp_path, prefix=name, ignore=found_set, ) # Handle remaining content found_set = self.watch_dir( tmp_path, prefix=name, ignore=found_set, seconds=-1, ) # Let the caller know our status if not sp.successful(): # Cleanup Temporary Path rm(tmp_path) return None if not len(found_set): return None # Create a resultset results = sortedset(key=lambda x: x.key()) # iterate through our found_set and create NNTPBinaryContent() # objects from them. part = 0 for path in found_set: # Iterate over our found files and determine their part # information _re_results = RAR_PART_RE.match(path) if _re_results: if _re_results.group('part') is not None: part = int(_re_results.group('part')) else: part += 1 else: part += 1 content = NNTPBinaryContent( path, part=part, total_parts=len(found_set), ) # Loaded data is by default detached; we want to attach it content.attach() # Add our attached content to our results results.add(content) # Clean our are list of objects to archive self.clear() # Return our return results
def decode(self, content=None, name=None, password=None, *args, **kwargs): """ content must be pointing to a directory containing rar files that can be easily sorted on. Alternatively, path can be of type NNTPContent() or a set/list of. If no password is specified, then the password configuration loaded into the class is used instead. An NNTPBinaryContent() object containing the contents of the package within a sortedset() object. All decoded() functions have to return a resultset() to be consistent with one another. """ if content is not None: self.add(content) # Some simple error checking to save from doing to much here if len(self) == 0: return None if not self.can_exe(self._unrar): return None if not password: password = self.password # Initialize our command execute = [ # Our Executable RAR Application self._unrar, # Use Add Flag 'x', # Assume Yes '-y', ] # Password Protection if password is not None: execute.append('-p%s' % password) else: # Do not prompt for password execute.append('-p-') if self.keep_broken: # Keep Broken Flag execute.append('-kb') if self.overwrite: # Overwrite files execute.append('-o+') else: # Don't overwrite files execute.append('-o-') if self.freshen: # Freshen files execute.append('-f') # Stop Switch Parsing execute.append('--') if not name: name = self.name if not name: name = random_str() for _path in self: # Temporary Path tmp_path, _ = self.mkstemp(content=name) with pushd(tmp_path): # Create our SubProcess Instance sp = SubProcess(list(execute) + [_path]) # Start our execution now sp.start() found_set = None while not sp.is_complete(timeout=1.5): found_set = self.watch_dir( tmp_path, ignore=found_set, ) # Handle remaining content found_set = self.watch_dir( tmp_path, ignore=found_set, seconds=-1, ) # Let the caller know our status if not sp.successful(): # Cleanup Temporary Path rm(tmp_path) return None if not len(found_set): logger.warning( 'RAR archive (%s) contained no content.' % basename(_path), ) # Clean our are list of objects to archive self.clear() # Return path containing unrar'ed content results = NNTPBinaryContent(tmp_path) # We intentionally attach it's content results.attach() # Create a sortedset to return _resultset = sortedset(key=lambda x: x.key()) _resultset.add(results) # Return our content return _resultset
def test(self, content=None, password=None): """ content must be pointing to a directory containing rar files that can be easily sorted on. Alternatively, path can be of type NNTPContent() or a set/list of. If no password is specified, then the password configuration loaded into the class is used instead. This function just tests an archive to see if it can be properly extracted using the known password. """ if content is not None: paths = self.get_paths(content) elif len(self.archive): # Get first item in archive paths = iter(self.archive) else: raise AttributeError("CodecRar: No rar file detected.") if not self.can_exe(self._unrar): return None if not password: password = self.password # Initialize our command execute = [ # Our Executable RAR Application self._unrar, # Use Test Flag 't', # Assume Yes '-y', ] # Password Protection if password is not None: execute.append('-p%s' % password) else: # Do not prompt for password execute.append('-p-') if self.keep_broken: # Keep Broken Flag execute.append('-kb') # Stop Switch Parsing execute.append('--') for _path in paths: # Create our SubProcess Instance sp = SubProcess(list(execute) + [_path]) # Start our execution now sp.start() sp.join() # Let the caller know our status if not sp.successful(): return False return True
def decode(self, content=None, *args, **kwargs): """ content must be pointing to a directory containing par files that can be easily retrieved. Alternatively, path can be of type NNTPContent() or a set/list of. An sortedset of NNTPBinaryContent() objects are returned containing any new content that was generated as a result of the par2 call If an error occurs then None is returned. """ if content is not None: self.add(content) # Some simple error checking to save from doing to much here if len(self) == 0: return None if not self.can_exe(self._par): return None # filter our results by indexes indexes = self.__filter_pars(self.archive, indexes=True, volumes=False) # Initialize our command execute = [ # Our Executable PAR Application self._par, # Use Repair 'repair', ] if self.cpu_cores is not None and self.cpu_cores > 1: # to repair concurrently - uses multiple threads execute.append('-t+') # Stop Switch Parsing execute.append('--') results = sortedset(key=lambda x: x.key()) for _path in indexes: # Get the directory the par file resides in par_path = dirname(_path) with pushd(par_path): # create a before snapshot before_snapshot = self.watch_dir( par_path, seconds=-1, ) # Create our SubProcess Instance sp = SubProcess(list(execute) + [basename(_path)]) # Start our execution now sp.start() # Track files after after_snapshot = sortedset() while not sp.is_complete(timeout=1.5): after_snapshot = self.watch_dir( par_path, ignore=after_snapshot, ) # Handle remaining content after_snapshot = self.watch_dir( par_path, ignore=after_snapshot, seconds=-1, ) # Add any new files detected to our result set otherwise we # just return an empty set total_parts = after_snapshot - before_snapshot for no, path in enumerate(total_parts): content = NNTPBinaryContent( path, part=no+1, total_parts=len(total_parts), ) # Loaded data is by default detached; we want to attach it content.attach() # Add our attached content to our results results.add(content) # Let the caller know our status if not sp.successful(): return None # Clean our are list of objects to archive self.clear() return results