def mainFooter(self):
print >> stderr
print >> stderr, "[+] End of search -- offset=%s (%s)" % (
self.current_offset // 8, humanFilesize(self.current_offset // 8))
size = (self.current_offset - self.start_offset) // 8
duration = time() - self.main_start
if 0.1 <= duration:
print >> stderr, "Total time: %s -- global rate: %s/sec" % (
humanDuration(duration * 1000), humanFilesize(
size // duration))
def mainFooter(self):
print >> stderr
print >> stderr, "[+] End of search -- offset=%s (%s)" % (
self.current_offset // 8,
humanFilesize(self.current_offset // 8),
)
size = (self.current_offset - self.start_offset) // 8
duration = time() - self.main_start
if 0.1 <= duration:
print >> stderr, "Total time: %s -- global rate: %s/sec" % (
humanDuration(duration * 1000),
humanFilesize(size // duration),
)
def createDescription(self):
superblock = self.getSuperblock()
block_size = 1024 << superblock["log_block_size"].value
nb_block = superblock["blocks_count"].value
total = nb_block * block_size
used = (superblock["free_blocks_count"].value) * block_size
desc = "EXT2/EXT3"
if "group[0]/inode_table/inode[7]/blocks" in self:
if 0 < self["group[0]/inode_table/inode[7]/blocks"].value:
desc = "EXT3"
else:
desc = "EXT2"
return desc + " file system: total=%s, used=%s, block=%s" % (
humanFilesize(total), humanFilesize(used),
humanFilesize(block_size))
def createDescription(self):
superblock = self.getSuperblock()
block_size = 1024 << superblock["log_block_size"].value
nb_block = superblock["blocks_count"].value
total = nb_block * block_size
used = (superblock["free_blocks_count"].value) * block_size
desc = "EXT2/EXT3"
if "group[0]/inode_table/inode[7]/blocks" in self:
if 0 < self["group[0]/inode_table/inode[7]/blocks"].value:
desc = "EXT3"
else:
desc = "EXT2"
return desc + " file system: total=%s, used=%s, block=%s" % (
humanFilesize(total), humanFilesize(used),
humanFilesize(block_size))
def processParser(self, offset, parser):
"""
Process a valid parser.
"""
text = "[+] File at %s" % (offset // 8)
if parser.content_size is not None:
text += " size=%s (%s)" % (parser.content_size // 8,
humanFilesize(parser.content_size // 8))
if not (parser.content_size
) or parser.content_size // 8 < FILE_MAX_SIZE:
text += ": " + parser.description
else:
text += ": " + parser.__class__.__name__
if self.output and parser.content_size:
if (offset == 0 and parser.content_size == self.size):
text += " (don't copy whole file)"
elif parser.content_size // 8 >= FILE_MAX_SIZE:
text += " (don't copy file, too big)"
elif not self.filter or self.filter(parser):
filename = self.output.createFilename(parser.filename_suffix)
filename = self.output.writeFile(filename, self.stream, offset,
parser.content_size)
text += " => %s" % filename
print text
self.next_progress = time() + PROGRESS_UPDATE
def main():
if len(sys.argv) != 2:
print >> sys.stderr, "usage: %s directory" % sys.argv[0]
sys.exit(1)
charset = getTerminalCharset()
directory = unicode(sys.argv[1], charset)
print "Download and check Hachoir testcase."
print
print "Use directory: %s" % directory
ok = testFiles(directory, TESTCASE_URL)
if not stringMD5("abc"):
print
for index in xrange(3):
print "!!! Warning: Python module md5 is missing, unable to check MD5 hash"
if ok:
print
totalsize = sum(item[1] for item in testcase_files)
print "Test case is ok (%s files, %s)" % (len(testcase_files),
humanFilesize(totalsize))
sys.exit(0)
else:
print
for index in xrange(3):
print "!!! ERROR !!!"
print
sys.exit(1)
def extractAVI(self, headers):
audio_index = 1
for stream in headers.array("stream"):
if "stream_hdr/stream_type" not in stream:
continue
stream_type = stream["stream_hdr/stream_type"].value
if stream_type == "vids":
if "stream_hdr" in stream:
meta = Metadata(self)
self.extractAVIVideo(stream["stream_hdr"], meta)
self.addGroup("video", meta, "Video stream")
elif stream_type == "auds":
if "stream_fmt" in stream:
meta = Metadata(self)
self.extractAVIAudio(stream["stream_fmt"], meta)
self.addGroup("audio[%u]" % audio_index, meta, "Audio stream")
audio_index += 1
if "avi_hdr" in headers:
self.useAviHeader(headers["avi_hdr"])
# Compute global bit rate
if self.has("duration") and "/movie/size" in headers:
self.bit_rate = float(headers["/movie/size"].value) * 8 / timedelta2seconds(self.get('duration'))
# Video has index?
if "/index" in headers:
self.comment = _("Has audio/video index (%s)") \
% humanFilesize(headers["/index"].size/8)
def __init__(self,
parent,
name,
length,
description=None,
parser=None,
filename=None,
mime_type=None,
parser_class=None):
if filename:
if not isinstance(filename, unicode):
filename = makePrintable(filename, "ISO-8859-1")
if not description:
description = 'File "%s" (%s)' % (filename,
humanFilesize(length))
Bytes.__init__(self, parent, name, length, description)
def createInputStream(cis, **args):
tags = args.setdefault("tags", [])
if parser_class:
tags.append(("class", parser_class))
if parser is not None:
tags.append(("id", parser.PARSER_TAGS["id"]))
if mime_type:
tags.append(("mime", mime_type))
if filename:
tags.append(("filename", filename))
return cis(**args)
self.setSubIStream(createInputStream)
def __call__(self, prev):
name = self.path + "[]"
address, size, last = self.cluster.next()
if self.filesize:
if self.done >= self.filesize:
error("(FAT) bad metadata for " + self.path)
return
field = File(self.root, name, size=size)
if prev.first is None:
field._description = 'File size: %s' % humanFilesize(self.filesize//8)
field.setSubIStream(self.createInputStream)
field.datasize = min(self.filesize - self.done, size)
self.done += field.datasize
else:
field = Directory(self.root, name, size=size)
padding = self.root.getFieldByAddress(address, feed=False)
if not isinstance(padding, (PaddingBytes, RawBytes)):
error("(FAT) address %u doesn't point to a padding field" % address)
return
if last:
next = None
else:
next = lambda: self(field)
field.setLinks(prev.first, next)
self.root.writeFieldsIn(padding, address, (field,))
return field
def extractAVI(self, headers):
audio_index = 1
for stream in headers.array("stream"):
if "stream_hdr/stream_type" not in stream:
continue
stream_type = stream["stream_hdr/stream_type"].value
if stream_type == "vids":
if "stream_hdr" in stream:
meta = Metadata(self)
self.extractAVIVideo(stream["stream_hdr"], meta)
self.addGroup("video", meta, "Video stream")
elif stream_type == "auds":
if "stream_fmt" in stream:
meta = Metadata(self)
self.extractAVIAudio(stream["stream_fmt"], meta)
self.addGroup("audio[%u]" % audio_index, meta,
"Audio stream")
audio_index += 1
if "avi_hdr" in headers:
self.useAviHeader(headers["avi_hdr"])
# Compute global bit rate
if self.has("duration") and "/movie/size" in headers:
self.bit_rate = float(
headers["/movie/size"].value) * 8 / timedelta2seconds(
self.get('duration'))
# Video has index?
if "/index" in headers:
self.comment = _("Has audio/video index (%s)") \
% humanFilesize(headers["/index"].size/8)
def main():
if len(sys.argv) != 2:
print >>sys.stderr, "usage: %s directory" % sys.argv[0]
sys.exit(1)
charset = getTerminalCharset()
directory = unicode(sys.argv[1], charset)
print "Download and check Hachoir testcase."
print
print "Use directory: %s" % directory
ok = testFiles(directory, TESTCASE_URL)
if not stringMD5("abc"):
print
for index in xrange(3):
print "!!! Warning: Python module md5 is missing, unable to check MD5 hash"
if ok:
print
totalsize = sum( item[1] for item in testcase_files )
print "Test case is ok (%s files, %s)" % (len(testcase_files), humanFilesize(totalsize))
sys.exit(0)
else:
print
for index in xrange(3):
print "!!! ERROR !!!"
print
sys.exit(1)
def createDescription(self):
if self["magic"].value == "S1SUSPEND\0":
text = "Suspend swap file version 1"
elif self["magic"].value == "SWAPSPACE2":
text = "Linux swap file version 2"
else:
text = "Linux swap file version 1"
nb_page = self.getPageCount()
return "%s, page size: %s, %s pages" % (text, humanFilesize(PAGE_SIZE), nb_page)
def createDescription(self):
desc = "Partition header: "
if self.isUsed():
system = self["system"].display
size = self["size"].value * BLOCK_SIZE
desc += "%s, %s" % (system, humanFilesize(size))
else:
desc += "(unused)"
return desc
def createDescription(self):
if self.isEmpty():
desc = "(terminator, empty header)"
else:
filename = self["name"].value
filesize = humanFilesize(self.getOctal("size"))
desc = "(%s: %s, %s)" % \
(filename, self["type"].display, filesize)
return "Tar File " + desc
def createDescription(self):
desc = "Partition header: "
if self.isUsed():
system = self["system"].display
size = self["size"].value * BLOCK_SIZE
desc += "%s, %s" % (system, humanFilesize(size))
else:
desc += "(unused)"
return desc
def createDescription(self):
if self.isEmpty():
desc = "(terminator, empty header)"
else:
filename = self["name"].value
filesize = humanFilesize(self.getOctal("size"))
desc = "(%s: %s, %s)" % \
(filename, self["type"].display, filesize)
return "Tar File " + desc
def createDescription(self):
if self["magic"].value == "S1SUSPEND\0":
text = "Suspend swap file version 1"
elif self["magic"].value == "SWAPSPACE2":
text = "Linux swap file version 2"
else:
text = "Linux swap file version 1"
nb_page = self.getPageCount()
return "%s, page size: %s, %s pages" % (text, humanFilesize(PAGE_SIZE),
nb_page)
def displayProgress(self):
"""
Display progress (to stdout) of the whole process.
Compute data rate (in byte per sec) and time estimation.
"""
# Program next update
self.next_progress = time() + PROGRESS_UPDATE
# Progress offset
percent = float(self.current_offset - self.start_offset) * 100 / (self.size - self.start_offset)
offset = self.current_offset // 8
message = "Search: %.2f%% -- offset=%u (%s)" % (percent, offset, humanFilesize(offset))
# Compute data rate (byte/sec)
average = self.datarate.average
if average:
message += " -- %s/sec " % humanFilesize(average // 8)
eta = float(self.size - self.current_offset) / average
message += " -- ETA: %s" % humanDuration(eta * 1000)
# Display message
print >> stderr, message
def mainHeader(self):
# Fix slice size if needed
self.slice_size = max(self.slice_size, self.patterns.max_length * 8)
# Load parsers if none has been choosen
if not self.patterns:
self.loadParsers()
bytes = (self.size - self.start_offset) // 8
print >> stderr, "[+] Start search on %s bytes (%s)" % (bytes, humanFilesize(bytes))
print >> stderr
self.stats = {}
self.current_offset = self.start_offset
self.main_start = time()
def download(url, filesize, md5sum, destname):
sys.stdout.write("[+] Download file %s (%s): " \
% (url, humanFilesize(filesize)))
sys.stdout.flush()
# Download data
request = Request(url)
try:
stream = urlopen(request)
except HTTPError, err:
if err.code == 404:
print "File not found (HTTP error 404)!"
else:
print "HTTP error (%s)" % unicode(err)
return False
def mainHeader(self):
# Fix slice size if needed
self.slice_size = max(self.slice_size, self.patterns.max_length * 8)
# Load parsers if none has been choosen
if not self.patterns:
self.loadParsers()
bytes = (self.size - self.start_offset) // 8
print >> stderr, "[+] Start search on %s bytes (%s)" % (
bytes, humanFilesize(bytes))
print >> stderr
self.stats = {}
self.current_offset = self.start_offset
self.main_start = time()
def download(url, filesize, md5sum, destname):
sys.stdout.write("[+] Download file %s (%s): " \
% (url, humanFilesize(filesize)))
sys.stdout.flush()
# Download data
request = Request(url)
try:
stream = urlopen(request)
except HTTPError, err:
if err.code == 404:
print "File not found (HTTP error 404)!"
else:
print "HTTP error (%s)" % unicode(err)
return False
def displayProgress(self):
"""
Display progress (to stdout) of the whole process.
Compute data rate (in byte per sec) and time estimation.
"""
# Program next update
self.next_progress = time() + PROGRESS_UPDATE
# Progress offset
percent = float(self.current_offset - self.start_offset) * 100 / (
self.size - self.start_offset)
offset = self.current_offset // 8
message = "Search: %.2f%% -- offset=%u (%s)" % (percent, offset,
humanFilesize(offset))
# Compute data rate (byte/sec)
average = self.datarate.average
if average:
message += " -- %s/sec " % humanFilesize(average // 8)
eta = float(self.size - self.current_offset) / average
message += " -- ETA: %s" % humanDuration(eta * 1000)
# Display message
print >> stderr, message
def createDescription(self):
desc = "Inode %s: " % self.uniq_id
size = self["size"].value
if self["blocks"].value == 0:
desc += "(unused)"
elif 11 <= self.uniq_id:
size = humanFilesize(size)
desc += "file, size=%s, mode=%s" % (size, self.getMode())
else:
if self.uniq_id in self.inode_type_name:
desc += self.inode_type_name[self.uniq_id]
if self.uniq_id == 2:
desc += " (%s)" % self.getMode()
else:
desc += "special"
return desc
def createDescription(self):
desc = "Inode %s: " % self.uniq_id
size = self["size"].value
if self["blocks"].value == 0:
desc += "(unused)"
elif 11 <= self.uniq_id:
size = humanFilesize(size)
desc += "file, size=%s, mode=%s" % (size, self.getMode())
else:
if self.uniq_id in self.inode_type_name:
desc += self.inode_type_name[self.uniq_id]
if self.uniq_id == 2:
desc += " (%s)" % self.getMode()
else:
desc += "special"
return desc
def checkPattern(self):
if not (config.check_padding_pattern):
return False
if self.pattern is None:
return False
if self.MAX_SIZE < self._size / 8:
self.info("only check first %s of padding" % humanFilesize(self.MAX_SIZE))
content = self._parent.stream.readBytes(self.absolute_address, self.MAX_SIZE)
else:
content = self.value
index = 0
pattern_len = len(self.pattern)
while index < len(content):
if content[index : index + pattern_len] != self.pattern:
self.warning("padding contents doesn't look normal" " (invalid pattern at byte %u)!" % index)
return False
index += pattern_len
return True
def __init__(self, parent, name, length, description=None,
parser=None, filename=None, mime_type=None, parser_class=None):
if filename:
if not isinstance(filename, unicode):
filename = makePrintable(filename, "ISO-8859-1")
if not description:
description = 'File "%s" (%s)' % (filename, humanFilesize(length))
Bytes.__init__(self, parent, name, length, description)
def createInputStream(cis, **args):
tags = args.setdefault("tags",[])
if parser_class:
tags.append(( "class", parser_class ))
if parser is not None:
tags.append(( "id", parser.PARSER_TAGS["id"] ))
if mime_type:
tags.append(( "mime", mime_type ))
if filename:
tags.append(( "filename", filename ))
return cis(**args)
self.setSubIStream(createInputStream)
def __init__(self, parent, name, description=None):
FieldSet.__init__(self, parent, name, description)
self._size = (self["size"].value + 3*4) * 8
if MAX_CHUNK_SIZE < (self._size//8):
raise ParserError("PNG: Chunk is too big (%s)"
% humanFilesize(self._size//8))
tag = self["tag"].value
self.desc_func = None
self.value_func = None
if tag in self.TAG_INFO:
self._name, self.parse_func, desc, value_func = self.TAG_INFO[tag]
if value_func:
self.value_func = value_func
self.createValue = self.createValueFunc
if desc:
if isinstance(desc, str):
self._description = desc
else:
self.desc_func = desc
else:
self._description = ""
self.parse_func = None
def checkPattern(self):
if not (config.check_padding_pattern):
return False
if self.pattern is None:
return False
if self.MAX_SIZE < self._size / 8:
self.info("only check first %s of padding" %
humanFilesize(self.MAX_SIZE))
content = self._parent.stream.readBytes(self.absolute_address,
self.MAX_SIZE)
else:
content = self.value
index = 0
pattern_len = len(self.pattern)
while index < len(content):
if content[index:index + pattern_len] != self.pattern:
self.warning("padding contents doesn't look normal"
" (invalid pattern at byte %u)!" % index)
return False
index += pattern_len
return True
def processParser(self, offset, parser):
"""
Process a valid parser.
"""
text = "[+] File at %s" % (offset // 8)
if parser.content_size is not None:
text += " size=%s (%s)" % (parser.content_size // 8, humanFilesize(parser.content_size // 8))
if not (parser.content_size) or parser.content_size // 8 < FILE_MAX_SIZE:
text += ": " + parser.description
else:
text += ": " + parser.__class__.__name__
if self.output and parser.content_size:
if offset == 0 and parser.content_size == self.size:
text += " (don't copy whole file)"
elif parser.content_size // 8 >= FILE_MAX_SIZE:
text += " (don't copy file, too big)"
elif not self.filter or self.filter(parser):
filename = self.output.createFilename(parser.filename_suffix)
filename = self.output.writeFile(filename, self.stream, offset, parser.content_size)
text += " => %s" % filename
print text
self.next_progress = time() + PROGRESS_UPDATE
def createDescription(self):
desc = "Group %s: %s" % (self.uniq_id, humanFilesize(self.size / 8))
if "superblock_copy" in self:
desc += " (with superblock copy)"
return desc
def createDescription(self):
desc = "Group %s: %s" % (self.uniq_id, humanFilesize(self.size/8))
if "superblock_copy" in self:
desc += " (with superblock copy)"
return desc
def createDisplay(self):
if not self["value"].hasValue():
return None
if self._name in ("length", "piece_length"):
return humanFilesize(self.value)
return FieldSet.createDisplay(self)
def createDescription(self):
size = self["nb_sectors"].value * self["bios/bytes_per_sector"].value
return "NTFS Master Boot Record (%s)" % humanFilesize(size)
def update(self, node):
if node.depth:
text = ' ' * (3 * node.depth - 2)
if node.childs:
text += '- '
elif node.field.is_field_set:
text += '+ '
else:
text += ' '
name = node.field.name
else:
text = ''
name = node.field.stream.source
if node.field.size:
if self.flags & self.use_absolute_address:
address = node.field.absolute_address
else:
address = node.field.address
display_bits = (address % 8) != 0 or (node.field.size % 8) != 0
if self.flags & self.hex_address:
if display_bits:
text += "%04x.%x" % (address/8, address%8)
else:
text += "%04x" % (address/8)
else:
if display_bits:
text += "%u.%u" % (address/8, address%8)
else:
text += "%u" % (address/8)
text += ") " + name
else:
text += "-> " + name
smart_display = True
if self.flags & self.display_value and node.field.hasValue():
if self.flags & self.human_size:
display = node.field.display
else:
display = node.field.raw_display
smart_display = False
text += "= %s" % display
if node.field.description and self.flags & self.display_description:
description = node.field.description
if not(self.flags & self.human_size):
description = makePrintable(description, "ASCII")
text += ": %s" % description
if self.flags & self.display_size and node.field.size or self.flags & self.display_type:
tmp_text = []
if self.flags & self.display_type:
tmp_text.append(node.field.getFieldType())
if self.flags & self.display_size:
if node.field.size % 8:
tmp_text.append( humanBitSize(node.field.size) )
else:
size = node.field.size / 8
if not self.flags & self.human_size:
tmp_text.append( ngettext("%u byte", "%u bytes", size) % size)
else:
tmp_text.append( humanFilesize(size) )
text += " (%s)" % ", ".join(tmp_text)
text = makePrintable(text, self.charset, to_unicode=True, smart=smart_display)
node.setText(text, self.flags)
def update(self, node):
if node.depth:
text = ' ' * (3 * node.depth - 2)
if node.childs:
text += '- '
elif node.field.is_field_set:
text += '+ '
else:
text += ' '
name = node.field.name
else:
text = ''
name = node.field.stream.source
if node.field.size:
if self.flags & self.use_absolute_address:
address = node.field.absolute_address
else:
address = node.field.address
display_bits = (address % 8) != 0 or (node.field.size % 8) != 0
if self.flags & self.hex_address:
if display_bits:
text += "%04x.%x" % (address/8, address%8)
else:
text += "%04x" % (address/8)
else:
if display_bits:
text += "%u.%u" % (address/8, address%8)
else:
text += "%u" % (address/8)
text += ") " + name
else:
text += "-> " + name
smart_display = True
if self.flags & self.display_value and node.field.hasValue():
if self.flags & self.human_size:
display = node.field.display
else:
display = node.field.raw_display
smart_display = False
text += "= %s" % display
if node.field.description and self.flags & self.display_description:
text += ": %s" % node.field.description
if self.flags & self.display_size and node.field.size or self.flags & self.display_type:
tmp_text = []
if self.flags & self.display_type:
tmp_text.append(node.field.getFieldType())
if self.flags & self.display_size:
if node.field.size % 8:
tmp_text.append( humanBitSize(node.field.size) )
else:
size = node.field.size / 8
if not self.flags & self.human_size:
tmp_text.append( ngettext("%u byte", "%u bytes", size) % size)
else:
tmp_text.append( humanFilesize(size) )
text += " (%s)" % ", ".join(tmp_text)
text = makePrintable(text, self.charset, to_unicode=True, smart=smart_display)
node.setText(text, self.flags)
def createDisplay(self):
if not self["value"].hasValue():
return None
if self._name in ("length", "piece_length"):
return humanFilesize(self.value)
return FieldSet.createDisplay(self)
def createDescription(self):
return "%d blocks of %s" % (self["num_blocks"].value,
humanFilesize(self["len"].value))
def createDescription(self):
return "%d blocks of %s" % (
self["num_blocks"].value, humanFilesize(self["len"].value))
def createDescription(self):
size = self["nb_sectors"].value * self["bios/bytes_per_sector"].value
return "NTFS Master Boot Record (%s)" % humanFilesize(size)