def test_program_that_will_not_die(self):
"""
SubProcess for a process that will never stop
"""
# Set up our threaded process to sleep for 1 day
sp = SubProcess(['sleep', '1d'])
# Now run our process
sp.start()
# Wait half a second (give thread time to start and run)
sp.join(timeout=0.5)
# PID is always None unless the process is still running
assert sp.pid() is not None
assert isinstance(sp.pid(), int)
# The process is still running
assert sp.is_complete() is False
# The process is not successful, but we shouldn't be relying on this
# value anyway because it's not even complete.
assert sp.successful() is False
# time should be elapsing
assert sp.elapsed() > 0.0
# No output to report at this time because one of the downsides of
# using subprocess is you can't retrieve the pipes content until the
# end. Thus these functions will simply return nothing.
assert len(sp.stdout()) > 0
assert len(sp.stderr()) > 0
# Currently the response code is still unknown
assert sp.response_code() is ReturnCode.Unknown
# We'll abort the process now
sp.abort()
# Wait for it to complete (because it will complete now)
sp.join()
# PID is always None when process is stopped
assert sp.pid() is None
# The response code should have change to Aborted
assert sp.response_code() is ReturnCode.Aborted
# The process is now complete
assert sp.is_complete() is True
# But we should not be successful at this point
assert sp.successful() is False
def test_program_that_will_not_die(self):
"""
SubProcess for a process that will never stop
"""
# Set up our threaded process to sleep for 1 day
sp = SubProcess(['sleep', '1d'])
# Now run our process
sp.start()
# Wait half a second (give thread time to start and run)
sp.join(timeout=0.5)
# PID is always None unless the process is still running
assert sp.pid() is not None
assert isinstance(sp.pid(), int)
# The process is still running
assert sp.is_complete() is False
# The process is not successful, but we shouldn't be relying on this
# value anyway because it's not even complete.
assert sp.successful() is False
# time should be elapsing
assert sp.elapsed() > 0.0
# No output to report at this time because one of the downsides of
# using subprocess is you can't retrieve the pipes content until the
# end. Thus these functions will simply return nothing.
assert len(sp.stdout()) > 0
assert len(sp.stderr()) > 0
# Currently the response code is still unknown
assert sp.response_code() is ReturnCode.Unknown
# We'll abort the process now
sp.abort()
# Wait for it to complete (because it will complete now)
sp.join()
# PID is always None when process is stopped
assert sp.pid() is None
# The response code should have change to Aborted
assert sp.response_code() is ReturnCode.Aborted
# The process is now complete
assert sp.is_complete() is True
# But we should not be successful at this point
assert sp.successful() is False
def test_simple_list(self):
"""
SubProcess for listing files
"""
# Set up our threaded process
sp = SubProcess(['ls', '-1', '/tmp'])
# Now run our process
sp.start()
# Wait for it to complete
sp.join()
assert sp.is_complete() is True
assert sp.response_code() is 0
assert sp.successful() is True
assert sp.elapsed() > 0.0
assert isinstance(sp.stdout(), list)
assert isinstance(sp.stderr(), list)
assert isinstance(sp.stdout(as_list=False), basestring)
assert isinstance(sp.stderr(as_list=False), basestring)
# Because under the hood stdout and stderr are stored into
# Stream objects, we need to be able to check that multple
# calls to the same object still return the same results
assert len(sp.stdout()) > 1
assert len(sp.stdout()) > 1
assert len(sp.stderr()) > 0
assert len(sp.stderr()) > 0
def test_program_timeout(self):
"""
SubProcess for a process that will be killed simply because it took
too long to execute.
"""
# Set up our threaded process to sleep for 1 day
# Bu
sp = SubProcess(['sleep', '1d'], timeout=1.0)
# Now run our process
sp.start()
# Wait until it expires
assert sp.is_complete(timeout=5.0) is True
# PID is always None unless the process is still running
assert sp.pid() is None
# The process is not successful, but we shouldn't be relying on this
# value anyway because it's not even complete.
assert sp.successful() is False
# time should be elapsing
assert sp.elapsed() > 0.0
# No output to report at this time because one of the downsides of
# using subprocess is you can't retrieve the pipes content until the
# end. Thus these functions will simply return nothing.
assert len(sp.stdout()) > 0
assert len(sp.stderr()) > 0
# A Timeout
assert sp.response_code() is ReturnCode.Timeout
def test_simple_list(self):
"""
SubProcess for listing files
"""
# Set up our threaded process
sp = SubProcess(['ls', '-1', '/tmp'])
# Now run our process
sp.start()
# Wait for it to complete
sp.join()
assert sp.is_complete() is True
assert sp.response_code() is 0
assert sp.successful() is True
assert sp.elapsed() > 0.0
assert isinstance(sp.stdout(), list)
assert isinstance(sp.stderr(), list)
assert isinstance(sp.stdout(as_list=False), basestring)
assert isinstance(sp.stderr(as_list=False), basestring)
# Because under the hood stdout and stderr are stored into
# Stream objects, we need to be able to check that multple
# calls to the same object still return the same results
assert len(sp.stdout()) > 1
assert len(sp.stdout()) > 1
assert len(sp.stderr()) > 0
assert len(sp.stderr()) > 0
def test_program_timeout(self):
"""
SubProcess for a process that will be killed simply because it took
too long to execute.
"""
# Set up our threaded process to sleep for 1 day
# Bu
sp = SubProcess(['sleep', '1d'], timeout=1.0)
# Now run our process
sp.start()
# Wait until it expires
assert sp.is_complete(timeout=5.0) is True
# PID is always None unless the process is still running
assert sp.pid() is None
# The process is not successful, but we shouldn't be relying on this
# value anyway because it's not even complete.
assert sp.successful() is False
# time should be elapsing
assert sp.elapsed() > 0.0
# No output to report at this time because one of the downsides of
# using subprocess is you can't retrieve the pipes content until the
# end. Thus these functions will simply return nothing.
assert len(sp.stdout()) > 0
assert len(sp.stderr()) > 0
# A Timeout
assert sp.response_code() is ReturnCode.Timeout
def test_bad_execution(self):
"""
SubProcess for listing files with bad arguments
"""
# Set up our threaded process
sp = SubProcess(['ls', '-wtf', '-not-a-valid-argument'])
# Now run our process
sp.start()
# Wait for it to complete
sp.join()
assert sp.is_complete() is True
assert sp.response_code() is not 0
assert sp.successful() is False
assert sp.elapsed() > 0.0
# Because under the hood stdout and stderr are stored into
# Stream objects, we need to be able to check that multple
# calls to the same object still return the same results
assert len(sp.stdout()) > 0
assert len(sp.stdout()) > 0
assert len(sp.stderr()) > 1
assert len(sp.stderr()) > 1
def test_bad_execution(self):
"""
SubProcess for listing files with bad arguments
"""
# Set up our threaded process
sp = SubProcess(['ls', '-wtf', '-not-a-valid-argument'])
# Now run our process
sp.start()
# Wait for it to complete
sp.join()
assert sp.is_complete() is True
assert sp.response_code() is not 0
assert sp.successful() is False
assert sp.elapsed() > 0.0
# Because under the hood stdout and stderr are stored into
# Stream objects, we need to be able to check that multple
# calls to the same object still return the same results
assert len(sp.stdout()) > 0
assert len(sp.stdout()) > 0
assert len(sp.stderr()) > 1
assert len(sp.stderr()) > 1
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing rar files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._unrar):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable RAR Application
self._unrar,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.keep_broken:
# Keep Broken Flag
execute.append('-kb')
if self.overwrite:
# Overwrite files
execute.append('-o+')
else:
# Don't overwrite files
execute.append('-o-')
if self.freshen:
# Freshen files
execute.append('-f')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'RAR archive (%s) contained no content.' %
basename(_path),
)
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def encode(self, content=None, name=None, *args, **kwargs):
"""
Takes a specified path (and or file) and compresses it. If this
function is successful, it returns a set of NNTPBinaryContent()
objects that are 'not' detached.
The function returns None if it fails in any way
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._rar):
return None
if not name:
name = self.name
if not name:
name = random_str()
tmp_path, tmp_file = self.mkstemp(content=name, suffix='.rar')
# Initialize our command
execute = [
# Our Executable RAR Application
self._rar,
# Use Add Flag
'a',
]
# Password Protection
if self.password is not None:
execute.append('-p%s' % self.password)
# Handle Compression Level
if self.level is CompressionLevel.Maximum:
execute.append('-m5')
elif self.level is CompressionLevel.Average:
execute.append('-m3')
elif self.level is CompressionLevel.Minimum:
execute.append('-m0')
# Exclude base directory from archive
execute.append('-ep1')
if not name:
name = splitext(basename(tmp_file))[0]
# Now place content within directory identifed by it's name
execute.append('-ap%s' % name)
# Handle RAR Volume Splitting
if self.volume_size:
execute.append('-v%sb' % self.volume_size)
# Handle Recovery Record
if self.recovery_record is not None:
execute.append('-rr%s' % self.recovery_record)
if self.cpu_cores is not None and self.cpu_cores > 1:
# create archive using multiple threads
execute.append('-mt%d' % self.cpu_cores)
# Stop Switch Parsing
execute.append('--')
# Specify the Destination Path
execute.append(tmp_file)
# Add all of our paths now
for _path in self:
execute.append(_path)
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
part = 0
for path in found_set:
# Iterate over our found files and determine their part
# information
_re_results = RAR_PART_RE.match(path)
if _re_results:
if _re_results.group('part') is not None:
part = int(_re_results.group('part'))
else:
part += 1
else:
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing 7-Zip files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.overwrite:
# Overwrite files
execute.append('-aoa')
else:
# Don't overwrite files
execute.append('-aos')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'7Z archive (%s) contained no content.' %
basename(_path), )
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def encode(self, content=None, name=None, *args, **kwargs):
"""
Takes a specified path (and or file) and compresses it. If this
function is successful, it returns a set of NNTPBinaryContent()
objects that are 'not' detached.
The function returns None if it fails in any way
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not name:
name = self.name
if not name:
name = random_str()
tmp_path, tmp_file = self.mkstemp(content=name, suffix='.7z')
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'a',
# Default mode is 7-Zip
'-t7z',
]
# Password Protection
if self.password is not None:
execute.append('-p%s' % self.password)
# Handle Compression Level
if self.level is CompressionLevel.Maximum:
execute.append('-mx9')
elif self.level is CompressionLevel.Average:
execute.append('-mx5')
elif self.level is CompressionLevel.Minimum:
execute.append('-mx1')
# Don't prompt for anything
execute.append('-y')
if not name:
name = splitext(basename(tmp_file))[0]
# Handle 7Z Volume Splitting
if self.volume_size:
execute.append('-v%sb' % self.volume_size)
if self.cpu_cores is not None and self.cpu_cores > 1:
# create archive using multiple threads
execute.append('-mmt%d' % self.cpu_cores)
# Stop Switch Parsing
execute.append('--')
# Specify the Destination Path
execute.append(tmp_file)
# Add all of our paths now
for _path in self:
execute.append(_path)
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
part = 0
for path in found_set:
# Iterate over our found files and determine their part
# information
_re_results = SEVEN_ZIP_PART_RE.match(path)
if _re_results:
if _re_results.group('part') is not None:
part = int(_re_results.group('part'))
elif _re_results.group('part0') is not None:
part = int(_re_results.group('part0'))
else:
part += 1
else:
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results
def decode(self, content=None, *args, **kwargs):
"""
content must be pointing to a directory containing par files that can
be easily retrieved. Alternatively, path can be of type NNTPContent()
or a set/list of.
An sortedset of NNTPBinaryContent() objects are returned containing
any new content that was generated as a result of the par2 call
If an error occurs then None is returned.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._par):
return None
# filter our results by indexes
indexes = self.__filter_pars(self.archive, indexes=True, volumes=False)
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Repair
'repair',
]
if self.cpu_cores is not None and self.cpu_cores > 1:
# to repair concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
results = sortedset(key=lambda x: x.key())
for _path in indexes:
# Get the directory the par file resides in
par_path = dirname(_path)
with pushd(par_path):
# create a before snapshot
before_snapshot = self.watch_dir(
par_path,
seconds=-1,
)
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [basename(_path)])
# Start our execution now
sp.start()
# Track files after
after_snapshot = sortedset()
while not sp.is_complete(timeout=1.5):
after_snapshot = self.watch_dir(
par_path,
ignore=after_snapshot,
)
# Handle remaining content
after_snapshot = self.watch_dir(
par_path,
ignore=after_snapshot,
seconds=-1,
)
# Add any new files detected to our result set otherwise we
# just return an empty set
total_parts = after_snapshot - before_snapshot
for no, path in enumerate(total_parts):
content = NNTPBinaryContent(
path,
part=no+1,
total_parts=len(total_parts),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Let the caller know our status
if not sp.successful():
return None
# Clean our are list of objects to archive
self.clear()
return results
def encode(self, content=None, *args, **kwargs):
"""
Takes a specified path (and or file) and creates par2 files based on
it. If this function is successful, it returns a set of
NNTPBinaryContent() objects identifying the PAR2 files generated
based on the passed in content.
The function returns None if it fails in any way.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._par):
return None
for target in self.archive:
# Base entry on first file in the list
name = basename(target)
target_dir = dirname(target)
# tmp_path, tmp_file = self.mkstemp(content=name, suffix='.par2')
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Create Flag
'create',
]
# Handle PAR Block Size
if self.block_size:
execute.append('-s%s' % self.block_size)
if self.recovery_percent:
execute.append('-r%d' % self.recovery_percent)
if self.cpu_cores is not None and self.cpu_cores > 1:
# to repair concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
# Now add our target (we can only do one at a time which i why we
# loop) and run our setups
execute.append(target)
found_set = sortedset()
with pushd(target_dir):
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
target_dir,
prefix=name,
regex=PAR_PART_RE,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
target_dir,
prefix=name,
regex=PAR_PART_RE,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# We're done; we failed
return None
if not len(found_set):
# We're done; we failed
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
part = 0
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
for path in found_set:
# Iterate over our found files and determine their part
# information
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results