def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing rar files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._unrar):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable RAR Application
self._unrar,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.keep_broken:
# Keep Broken Flag
execute.append('-kb')
if self.overwrite:
# Overwrite files
execute.append('-o+')
else:
# Don't overwrite files
execute.append('-o-')
if self.freshen:
# Freshen files
execute.append('-f')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'RAR archive (%s) contained no content.' %
basename(_path),
)
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def encode(self, content=None, name=None, *args, **kwargs):
"""
Takes a specified path (and or file) and compresses it. If this
function is successful, it returns a set of NNTPBinaryContent()
objects that are 'not' detached.
The function returns None if it fails in any way
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._rar):
return None
if not name:
name = self.name
if not name:
name = random_str()
tmp_path, tmp_file = self.mkstemp(content=name, suffix='.rar')
# Initialize our command
execute = [
# Our Executable RAR Application
self._rar,
# Use Add Flag
'a',
]
# Password Protection
if self.password is not None:
execute.append('-p%s' % self.password)
# Handle Compression Level
if self.level is CompressionLevel.Maximum:
execute.append('-m5')
elif self.level is CompressionLevel.Average:
execute.append('-m3')
elif self.level is CompressionLevel.Minimum:
execute.append('-m0')
# Exclude base directory from archive
execute.append('-ep1')
if not name:
name = splitext(basename(tmp_file))[0]
# Now place content within directory identifed by it's name
execute.append('-ap%s' % name)
# Handle RAR Volume Splitting
if self.volume_size:
execute.append('-v%sb' % self.volume_size)
# Handle Recovery Record
if self.recovery_record is not None:
execute.append('-rr%s' % self.recovery_record)
if self.cpu_cores is not None and self.cpu_cores > 1:
# create archive using multiple threads
execute.append('-mt%d' % self.cpu_cores)
# Stop Switch Parsing
execute.append('--')
# Specify the Destination Path
execute.append(tmp_file)
# Add all of our paths now
for _path in self:
execute.append(_path)
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
part = 0
for path in found_set:
# Iterate over our found files and determine their part
# information
_re_results = RAR_PART_RE.match(path)
if _re_results:
if _re_results.group('part') is not None:
part = int(_re_results.group('part'))
else:
part += 1
else:
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing 7-Zip files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.overwrite:
# Overwrite files
execute.append('-aoa')
else:
# Don't overwrite files
execute.append('-aos')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'7Z archive (%s) contained no content.' %
basename(_path), )
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def encode(self, content=None, name=None, *args, **kwargs):
"""
Takes a specified path (and or file) and compresses it. If this
function is successful, it returns a set of NNTPBinaryContent()
objects that are 'not' detached.
The function returns None if it fails in any way
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not name:
name = self.name
if not name:
name = random_str()
tmp_path, tmp_file = self.mkstemp(content=name, suffix='.7z')
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'a',
# Default mode is 7-Zip
'-t7z',
]
# Password Protection
if self.password is not None:
execute.append('-p%s' % self.password)
# Handle Compression Level
if self.level is CompressionLevel.Maximum:
execute.append('-mx9')
elif self.level is CompressionLevel.Average:
execute.append('-mx5')
elif self.level is CompressionLevel.Minimum:
execute.append('-mx1')
# Don't prompt for anything
execute.append('-y')
if not name:
name = splitext(basename(tmp_file))[0]
# Handle 7Z Volume Splitting
if self.volume_size:
execute.append('-v%sb' % self.volume_size)
if self.cpu_cores is not None and self.cpu_cores > 1:
# create archive using multiple threads
execute.append('-mmt%d' % self.cpu_cores)
# Stop Switch Parsing
execute.append('--')
# Specify the Destination Path
execute.append(tmp_file)
# Add all of our paths now
for _path in self:
execute.append(_path)
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
part = 0
for path in found_set:
# Iterate over our found files and determine their part
# information
_re_results = SEVEN_ZIP_PART_RE.match(path)
if _re_results:
if _re_results.group('part') is not None:
part = int(_re_results.group('part'))
elif _re_results.group('part0') is not None:
part = int(_re_results.group('part0'))
else:
part += 1
else:
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results
def decode(self, content=None, *args, **kwargs):
"""
content must be pointing to a directory containing par files that can
be easily retrieved. Alternatively, path can be of type NNTPContent()
or a set/list of.
An sortedset of NNTPBinaryContent() objects are returned containing
any new content that was generated as a result of the par2 call
If an error occurs then None is returned.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._par):
return None
# filter our results by indexes
indexes = self.__filter_pars(self.archive, indexes=True, volumes=False)
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Repair
'repair',
]
if self.cpu_cores is not None and self.cpu_cores > 1:
# to repair concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
results = sortedset(key=lambda x: x.key())
for _path in indexes:
# Get the directory the par file resides in
par_path = dirname(_path)
with pushd(par_path):
# create a before snapshot
before_snapshot = self.watch_dir(
par_path,
seconds=-1,
)
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [basename(_path)])
# Start our execution now
sp.start()
# Track files after
after_snapshot = sortedset()
while not sp.is_complete(timeout=1.5):
after_snapshot = self.watch_dir(
par_path,
ignore=after_snapshot,
)
# Handle remaining content
after_snapshot = self.watch_dir(
par_path,
ignore=after_snapshot,
seconds=-1,
)
# Add any new files detected to our result set otherwise we
# just return an empty set
total_parts = after_snapshot - before_snapshot
for no, path in enumerate(total_parts):
content = NNTPBinaryContent(
path,
part=no+1,
total_parts=len(total_parts),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Let the caller know our status
if not sp.successful():
return None
# Clean our are list of objects to archive
self.clear()
return results
def encode(self, content=None, *args, **kwargs):
"""
Takes a specified path (and or file) and creates par2 files based on
it. If this function is successful, it returns a set of
NNTPBinaryContent() objects identifying the PAR2 files generated
based on the passed in content.
The function returns None if it fails in any way.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._par):
return None
for target in self.archive:
# Base entry on first file in the list
name = basename(target)
target_dir = dirname(target)
# tmp_path, tmp_file = self.mkstemp(content=name, suffix='.par2')
# Initialize our command
execute = [
# Our Executable PAR Application
self._par,
# Use Create Flag
'create',
]
# Handle PAR Block Size
if self.block_size:
execute.append('-s%s' % self.block_size)
if self.recovery_percent:
execute.append('-r%d' % self.recovery_percent)
if self.cpu_cores is not None and self.cpu_cores > 1:
# to repair concurrently - uses multiple threads
execute.append('-t+')
# Stop Switch Parsing
execute.append('--')
# Now add our target (we can only do one at a time which i why we
# loop) and run our setups
execute.append(target)
found_set = sortedset()
with pushd(target_dir):
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
target_dir,
prefix=name,
regex=PAR_PART_RE,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
target_dir,
prefix=name,
regex=PAR_PART_RE,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# We're done; we failed
return None
if not len(found_set):
# We're done; we failed
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
part = 0
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
for path in found_set:
# Iterate over our found files and determine their part
# information
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results