def mkstemp(self, content=None, suffix='.tmp', prefix='_tmp_'):
"""
A wrapper to mkstemp that only handles reference to the filepath/name
itself. It creates a unique subdirectory that it generates the new
temporary file within that can be referenced.
If a content is specified, then the function parses out the directory
infront of it and possibly a prefix at the end and swaps it with the
prefix specified. This is just an easier way of manipulating a
filename or directory name that was recently pulled from an
NNTPContent() object.
This function returns both the temporary directory created and the
temporary file prepared.
"""
# Create a temporary directory to work in
tmp_path = mkdtemp(prefix='_nr.codec-', dir=self.work_dir)
tmp_file = None
if isinstance(content, basestring):
tmp_file = join(tmp_path, '%s%s' % (
basename(content),
suffix,
))
elif isinstance(content, NNTPContent):
# use the filename based on the path
if content.filename:
tmp_file = join(
tmp_path, '%s%s' % (
splitext(basename(content.filename))[0],
suffix,
))
elif isinstance(content, NNTPArticle):
if len(content) > 0:
if content[0].filename:
tmp_file = join(
tmp_path, '%s%s' % (
splitext(basename(content[0].filename))[0],
suffix,
))
if tmp_file is None:
# Fall back
tmp_file = join(
tmp_path,
'%s%s' % (random_str(), suffix),
)
return tmp_path, tmp_file
def mkstemp(self, content=None, suffix='.tmp', prefix='_tmp_'):
"""
A wrapper to mkstemp that only handles reference to the filepath/name
itself. It creates a unique subdirectory that it generates the new
temporary file within that can be referenced.
If a content is specified, then the function parses out the directory
infront of it and possibly a prefix at the end and swaps it with the
prefix specified. This is just an easier way of manipulating a
filename or directory name that was recently pulled from an
NNTPContent() object.
This function returns both the temporary directory created and the
temporary file prepared.
"""
# Create a temporary directory to work in
tmp_path = mkdtemp(prefix='_nr.codec-', dir=self.work_dir)
tmp_file = None
if isinstance(content, basestring):
tmp_file = join(
tmp_path,
'%s%s' % (basename(content), suffix,
))
elif isinstance(content, NNTPContent):
# use the filename based on the path
if content.filename:
tmp_file = join(
tmp_path,
'%s%s' % (splitext(basename(content.filename))[0], suffix,
))
elif isinstance(content, NNTPArticle):
if len(content) > 0:
if content[0].filename:
tmp_file = join(
tmp_path,
'%s%s' % (
splitext(basename(content[0].filename))[0],
suffix,
))
if tmp_file is None:
# Fall back
tmp_file = join(
tmp_path,
'%s%s' % (random_str(), suffix),
)
return tmp_path, tmp_file
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing rar files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._unrar):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable RAR Application
self._unrar,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.keep_broken:
# Keep Broken Flag
execute.append('-kb')
if self.overwrite:
# Overwrite files
execute.append('-o+')
else:
# Don't overwrite files
execute.append('-o-')
if self.freshen:
# Freshen files
execute.append('-f')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'RAR archive (%s) contained no content.' %
basename(_path),
)
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def encode(self, content=None, name=None, *args, **kwargs):
"""
Takes a specified path (and or file) and compresses it. If this
function is successful, it returns a set of NNTPBinaryContent()
objects that are 'not' detached.
The function returns None if it fails in any way
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._rar):
return None
if not name:
name = self.name
if not name:
name = random_str()
tmp_path, tmp_file = self.mkstemp(content=name, suffix='.rar')
# Initialize our command
execute = [
# Our Executable RAR Application
self._rar,
# Use Add Flag
'a',
]
# Password Protection
if self.password is not None:
execute.append('-p%s' % self.password)
# Handle Compression Level
if self.level is CompressionLevel.Maximum:
execute.append('-m5')
elif self.level is CompressionLevel.Average:
execute.append('-m3')
elif self.level is CompressionLevel.Minimum:
execute.append('-m0')
# Exclude base directory from archive
execute.append('-ep1')
if not name:
name = splitext(basename(tmp_file))[0]
# Now place content within directory identifed by it's name
execute.append('-ap%s' % name)
# Handle RAR Volume Splitting
if self.volume_size:
execute.append('-v%sb' % self.volume_size)
# Handle Recovery Record
if self.recovery_record is not None:
execute.append('-rr%s' % self.recovery_record)
if self.cpu_cores is not None and self.cpu_cores > 1:
# create archive using multiple threads
execute.append('-mt%d' % self.cpu_cores)
# Stop Switch Parsing
execute.append('--')
# Specify the Destination Path
execute.append(tmp_file)
# Add all of our paths now
for _path in self:
execute.append(_path)
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
part = 0
for path in found_set:
# Iterate over our found files and determine their part
# information
_re_results = RAR_PART_RE.match(path)
if _re_results:
if _re_results.group('part') is not None:
part = int(_re_results.group('part'))
else:
part += 1
else:
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results
def decode(self, content=None, name=None, password=None, *args, **kwargs):
"""
content must be pointing to a directory containing 7-Zip files that can
be easily sorted on. Alternatively, path can be of type NNTPContent()
or a set/list of.
If no password is specified, then the password configuration loaded
into the class is used instead.
An NNTPBinaryContent() object containing the contents of the package
within a sortedset() object. All decoded() functions have to return
a resultset() to be consistent with one another.
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not password:
password = self.password
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'x',
# Assume Yes
'-y',
]
# Password Protection
if password is not None:
execute.append('-p%s' % password)
else:
# Do not prompt for password
execute.append('-p-')
if self.overwrite:
# Overwrite files
execute.append('-aoa')
else:
# Don't overwrite files
execute.append('-aos')
# Stop Switch Parsing
execute.append('--')
if not name:
name = self.name
if not name:
name = random_str()
for _path in self:
# Temporary Path
tmp_path, _ = self.mkstemp(content=name)
with pushd(tmp_path):
# Create our SubProcess Instance
sp = SubProcess(list(execute) + [_path])
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
logger.warning(
'7Z archive (%s) contained no content.' %
basename(_path), )
# Clean our are list of objects to archive
self.clear()
# Return path containing unrar'ed content
results = NNTPBinaryContent(tmp_path)
# We intentionally attach it's content
results.attach()
# Create a sortedset to return
_resultset = sortedset(key=lambda x: x.key())
_resultset.add(results)
# Return our content
return _resultset
def encode(self, content=None, name=None, *args, **kwargs):
"""
Takes a specified path (and or file) and compresses it. If this
function is successful, it returns a set of NNTPBinaryContent()
objects that are 'not' detached.
The function returns None if it fails in any way
"""
if content is not None:
self.add(content)
# Some simple error checking to save from doing to much here
if len(self) == 0:
return None
if not self.can_exe(self._bin):
return None
if not name:
name = self.name
if not name:
name = random_str()
tmp_path, tmp_file = self.mkstemp(content=name, suffix='.7z')
# Initialize our command
execute = [
# Our Executable 7-Zip Application
self._bin,
# Use Add Flag
'a',
# Default mode is 7-Zip
'-t7z',
]
# Password Protection
if self.password is not None:
execute.append('-p%s' % self.password)
# Handle Compression Level
if self.level is CompressionLevel.Maximum:
execute.append('-mx9')
elif self.level is CompressionLevel.Average:
execute.append('-mx5')
elif self.level is CompressionLevel.Minimum:
execute.append('-mx1')
# Don't prompt for anything
execute.append('-y')
if not name:
name = splitext(basename(tmp_file))[0]
# Handle 7Z Volume Splitting
if self.volume_size:
execute.append('-v%sb' % self.volume_size)
if self.cpu_cores is not None and self.cpu_cores > 1:
# create archive using multiple threads
execute.append('-mmt%d' % self.cpu_cores)
# Stop Switch Parsing
execute.append('--')
# Specify the Destination Path
execute.append(tmp_file)
# Add all of our paths now
for _path in self:
execute.append(_path)
# Create our SubProcess Instance
sp = SubProcess(execute)
# Start our execution now
sp.start()
found_set = None
while not sp.is_complete(timeout=1.5):
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
)
# Handle remaining content
found_set = self.watch_dir(
tmp_path,
prefix=name,
ignore=found_set,
seconds=-1,
)
# Let the caller know our status
if not sp.successful():
# Cleanup Temporary Path
rm(tmp_path)
return None
if not len(found_set):
return None
# Create a resultset
results = sortedset(key=lambda x: x.key())
# iterate through our found_set and create NNTPBinaryContent()
# objects from them.
part = 0
for path in found_set:
# Iterate over our found files and determine their part
# information
_re_results = SEVEN_ZIP_PART_RE.match(path)
if _re_results:
if _re_results.group('part') is not None:
part = int(_re_results.group('part'))
elif _re_results.group('part0') is not None:
part = int(_re_results.group('part0'))
else:
part += 1
else:
part += 1
content = NNTPBinaryContent(
path,
part=part,
total_parts=len(found_set),
)
# Loaded data is by default detached; we want to attach it
content.attach()
# Add our attached content to our results
results.add(content)
# Clean our are list of objects to archive
self.clear()
# Return our
return results
