Example #1
0
 def mainFooter(self):
     print >> stderr
     print >> stderr, "[+] End of search -- offset=%s (%s)" % (
         self.current_offset // 8, humanFilesize(self.current_offset // 8))
     size = (self.current_offset - self.start_offset) // 8
     duration = time() - self.main_start
     if 0.1 <= duration:
         print >> stderr, "Total time: %s -- global rate: %s/sec" % (
             humanDuration(duration * 1000), humanFilesize(
                 size // duration))
Example #2
0
 def mainFooter(self):
     print >> stderr
     print >> stderr, "[+] End of search -- offset=%s (%s)" % (
         self.current_offset // 8,
         humanFilesize(self.current_offset // 8),
     )
     size = (self.current_offset - self.start_offset) // 8
     duration = time() - self.main_start
     if 0.1 <= duration:
         print >> stderr, "Total time: %s -- global rate: %s/sec" % (
             humanDuration(duration * 1000),
             humanFilesize(size // duration),
         )
 def createDescription(self):
     superblock = self.getSuperblock()
     block_size = 1024 << superblock["log_block_size"].value
     nb_block = superblock["blocks_count"].value
     total = nb_block * block_size
     used = (superblock["free_blocks_count"].value) * block_size
     desc = "EXT2/EXT3"
     if "group[0]/inode_table/inode[7]/blocks" in self:
         if 0 < self["group[0]/inode_table/inode[7]/blocks"].value:
             desc = "EXT3"
         else:
             desc = "EXT2"
     return desc + " file system: total=%s, used=%s, block=%s" % (
         humanFilesize(total), humanFilesize(used),
         humanFilesize(block_size))
Example #4
0
 def createDescription(self):
     superblock = self.getSuperblock()
     block_size = 1024 << superblock["log_block_size"].value
     nb_block = superblock["blocks_count"].value
     total = nb_block * block_size
     used = (superblock["free_blocks_count"].value) * block_size
     desc = "EXT2/EXT3"
     if "group[0]/inode_table/inode[7]/blocks" in self:
         if 0 < self["group[0]/inode_table/inode[7]/blocks"].value:
             desc = "EXT3"
         else:
             desc = "EXT2"
     return desc + " file system: total=%s, used=%s, block=%s" % (
         humanFilesize(total), humanFilesize(used),
         humanFilesize(block_size))
Example #5
0
    def processParser(self, offset, parser):
        """
        Process a valid parser.
        """
        text = "[+] File at %s" % (offset // 8)
        if parser.content_size is not None:
            text += " size=%s (%s)" % (parser.content_size // 8,
                                       humanFilesize(parser.content_size // 8))
        if not (parser.content_size
                ) or parser.content_size // 8 < FILE_MAX_SIZE:
            text += ": " + parser.description
        else:
            text += ": " + parser.__class__.__name__

        if self.output and parser.content_size:
            if (offset == 0 and parser.content_size == self.size):
                text += " (don't copy whole file)"
            elif parser.content_size // 8 >= FILE_MAX_SIZE:
                text += " (don't copy file, too big)"
            elif not self.filter or self.filter(parser):
                filename = self.output.createFilename(parser.filename_suffix)
                filename = self.output.writeFile(filename, self.stream, offset,
                                                 parser.content_size)
                text += " => %s" % filename
        print text
        self.next_progress = time() + PROGRESS_UPDATE
def main():
    if len(sys.argv) != 2:
        print >> sys.stderr, "usage: %s directory" % sys.argv[0]
        sys.exit(1)
    charset = getTerminalCharset()
    directory = unicode(sys.argv[1], charset)

    print "Download and check Hachoir testcase."
    print
    print "Use directory: %s" % directory
    ok = testFiles(directory, TESTCASE_URL)
    if not stringMD5("abc"):
        print
        for index in xrange(3):
            print "!!! Warning: Python module md5 is missing, unable to check MD5 hash"
    if ok:
        print
        totalsize = sum(item[1] for item in testcase_files)
        print "Test case is ok (%s files, %s)" % (len(testcase_files),
                                                  humanFilesize(totalsize))
        sys.exit(0)
    else:
        print
        for index in xrange(3):
            print "!!! ERROR !!!"
        print
        sys.exit(1)
Example #7
0
    def extractAVI(self, headers):
        audio_index = 1
        for stream in headers.array("stream"):
            if "stream_hdr/stream_type" not in stream:
                continue
            stream_type = stream["stream_hdr/stream_type"].value
            if stream_type == "vids":
                if "stream_hdr" in stream:
                    meta = Metadata(self)
                    self.extractAVIVideo(stream["stream_hdr"], meta)
                    self.addGroup("video", meta, "Video stream")
            elif stream_type == "auds":
                if "stream_fmt" in stream:
                    meta = Metadata(self)
                    self.extractAVIAudio(stream["stream_fmt"], meta)
                    self.addGroup("audio[%u]" % audio_index, meta, "Audio stream")
                    audio_index += 1
        if "avi_hdr" in headers:
            self.useAviHeader(headers["avi_hdr"])

        # Compute global bit rate
        if self.has("duration") and "/movie/size" in headers:
            self.bit_rate = float(headers["/movie/size"].value) * 8 / timedelta2seconds(self.get('duration'))

        # Video has index?
        if "/index" in headers:
            self.comment = _("Has audio/video index (%s)") \
                % humanFilesize(headers["/index"].size/8)
Example #8
0
    def __init__(self,
                 parent,
                 name,
                 length,
                 description=None,
                 parser=None,
                 filename=None,
                 mime_type=None,
                 parser_class=None):
        if filename:
            if not isinstance(filename, unicode):
                filename = makePrintable(filename, "ISO-8859-1")
            if not description:
                description = 'File "%s" (%s)' % (filename,
                                                  humanFilesize(length))
        Bytes.__init__(self, parent, name, length, description)

        def createInputStream(cis, **args):
            tags = args.setdefault("tags", [])
            if parser_class:
                tags.append(("class", parser_class))
            if parser is not None:
                tags.append(("id", parser.PARSER_TAGS["id"]))
            if mime_type:
                tags.append(("mime", mime_type))
            if filename:
                tags.append(("filename", filename))
            return cis(**args)

        self.setSubIStream(createInputStream)
Example #9
0
 def __call__(self, prev):
     name = self.path + "[]"
     address, size, last = self.cluster.next()
     if self.filesize:
         if self.done >= self.filesize:
             error("(FAT) bad metadata for " + self.path)
             return
         field = File(self.root, name, size=size)
         if prev.first is None:
             field._description = 'File size: %s' % humanFilesize(self.filesize//8)
             field.setSubIStream(self.createInputStream)
         field.datasize = min(self.filesize - self.done, size)
         self.done += field.datasize
     else:
         field = Directory(self.root, name, size=size)
     padding = self.root.getFieldByAddress(address, feed=False)
     if not isinstance(padding, (PaddingBytes, RawBytes)):
         error("(FAT) address %u doesn't point to a padding field" % address)
         return
     if last:
         next = None
     else:
         next = lambda: self(field)
     field.setLinks(prev.first, next)
     self.root.writeFieldsIn(padding, address, (field,))
     return field
Example #10
0
    def extractAVI(self, headers):
        audio_index = 1
        for stream in headers.array("stream"):
            if "stream_hdr/stream_type" not in stream:
                continue
            stream_type = stream["stream_hdr/stream_type"].value
            if stream_type == "vids":
                if "stream_hdr" in stream:
                    meta = Metadata(self)
                    self.extractAVIVideo(stream["stream_hdr"], meta)
                    self.addGroup("video", meta, "Video stream")
            elif stream_type == "auds":
                if "stream_fmt" in stream:
                    meta = Metadata(self)
                    self.extractAVIAudio(stream["stream_fmt"], meta)
                    self.addGroup("audio[%u]" % audio_index, meta,
                                  "Audio stream")
                    audio_index += 1
        if "avi_hdr" in headers:
            self.useAviHeader(headers["avi_hdr"])

        # Compute global bit rate
        if self.has("duration") and "/movie/size" in headers:
            self.bit_rate = float(
                headers["/movie/size"].value) * 8 / timedelta2seconds(
                    self.get('duration'))

        # Video has index?
        if "/index" in headers:
            self.comment = _("Has audio/video index (%s)") \
                % humanFilesize(headers["/index"].size/8)
Example #11
0
def main():
    if len(sys.argv) != 2:
        print >>sys.stderr, "usage: %s directory" % sys.argv[0]
        sys.exit(1)
    charset = getTerminalCharset()
    directory = unicode(sys.argv[1], charset)

    print "Download and check Hachoir testcase."
    print
    print "Use directory: %s" % directory
    ok = testFiles(directory, TESTCASE_URL)
    if not stringMD5("abc"):
        print
        for index in xrange(3):
            print "!!! Warning: Python module md5 is missing, unable to check MD5 hash"
    if ok:
        print
        totalsize = sum( item[1] for item in testcase_files )
        print "Test case is ok (%s files, %s)" % (len(testcase_files), humanFilesize(totalsize))
        sys.exit(0)
    else:
        print
        for index in xrange(3):
            print "!!! ERROR !!!"
        print
        sys.exit(1)
Example #12
0
 def createDescription(self):
     if self["magic"].value == "S1SUSPEND\0":
         text = "Suspend swap file version 1"
     elif self["magic"].value == "SWAPSPACE2":
         text = "Linux swap file version 2"
     else:
         text = "Linux swap file version 1"
     nb_page = self.getPageCount()
     return "%s, page size: %s, %s pages" % (text, humanFilesize(PAGE_SIZE), nb_page)
Example #13
0
 def createDescription(self):
     desc = "Partition header: "
     if self.isUsed():
         system = self["system"].display
         size = self["size"].value * BLOCK_SIZE
         desc += "%s, %s" % (system, humanFilesize(size))
     else:
         desc += "(unused)"
     return desc
Example #14
0
 def createDescription(self):
     if self.isEmpty():
         desc = "(terminator, empty header)"
     else:
         filename = self["name"].value
         filesize = humanFilesize(self.getOctal("size"))
         desc = "(%s: %s, %s)" % \
             (filename, self["type"].display, filesize)
     return "Tar File " + desc
Example #15
0
 def createDescription(self):
     desc = "Partition header: "
     if self.isUsed():
         system = self["system"].display
         size = self["size"].value * BLOCK_SIZE
         desc += "%s, %s" % (system, humanFilesize(size))
     else:
         desc += "(unused)"
     return desc
Example #16
0
 def createDescription(self):
     if self.isEmpty():
         desc = "(terminator, empty header)"
     else:
         filename = self["name"].value
         filesize = humanFilesize(self.getOctal("size"))
         desc = "(%s: %s, %s)" % \
             (filename, self["type"].display, filesize)
     return "Tar File " + desc
Example #17
0
 def createDescription(self):
     if self["magic"].value == "S1SUSPEND\0":
         text = "Suspend swap file version 1"
     elif self["magic"].value == "SWAPSPACE2":
         text = "Linux swap file version 2"
     else:
         text = "Linux swap file version 1"
     nb_page = self.getPageCount()
     return "%s, page size: %s, %s pages" % (text, humanFilesize(PAGE_SIZE),
                                             nb_page)
Example #18
0
    def displayProgress(self):
        """
        Display progress (to stdout) of the whole process.
        Compute data rate (in byte per sec) and time estimation.
        """
        # Program next update
        self.next_progress = time() + PROGRESS_UPDATE

        # Progress offset
        percent = float(self.current_offset - self.start_offset) * 100 / (self.size - self.start_offset)
        offset = self.current_offset // 8
        message = "Search: %.2f%% -- offset=%u (%s)" % (percent, offset, humanFilesize(offset))

        # Compute data rate (byte/sec)
        average = self.datarate.average
        if average:
            message += " -- %s/sec " % humanFilesize(average // 8)
            eta = float(self.size - self.current_offset) / average
            message += " -- ETA: %s" % humanDuration(eta * 1000)

        # Display message
        print >> stderr, message
Example #19
0
    def mainHeader(self):
        # Fix slice size if needed
        self.slice_size = max(self.slice_size, self.patterns.max_length * 8)

        # Load parsers if none has been choosen
        if not self.patterns:
            self.loadParsers()

        bytes = (self.size - self.start_offset) // 8
        print >> stderr, "[+] Start search on %s bytes (%s)" % (bytes, humanFilesize(bytes))
        print >> stderr
        self.stats = {}
        self.current_offset = self.start_offset
        self.main_start = time()
def download(url, filesize, md5sum, destname):
    sys.stdout.write("[+] Download file %s (%s): " \
        % (url, humanFilesize(filesize)))
    sys.stdout.flush()

    # Download data
    request = Request(url)
    try:
        stream = urlopen(request)
    except HTTPError, err:
        if err.code == 404:
            print "File not found (HTTP error 404)!"
        else:
            print "HTTP error (%s)" % unicode(err)
        return False
Example #21
0
    def mainHeader(self):
        # Fix slice size if needed
        self.slice_size = max(self.slice_size, self.patterns.max_length * 8)

        # Load parsers if none has been choosen
        if not self.patterns:
            self.loadParsers()

        bytes = (self.size - self.start_offset) // 8
        print >> stderr, "[+] Start search on %s bytes (%s)" % (
            bytes, humanFilesize(bytes))
        print >> stderr
        self.stats = {}
        self.current_offset = self.start_offset
        self.main_start = time()
Example #22
0
def download(url, filesize, md5sum, destname):
    sys.stdout.write("[+] Download file %s (%s): " \
        % (url, humanFilesize(filesize)))
    sys.stdout.flush()

    # Download data
    request = Request(url)
    try:
        stream = urlopen(request)
    except HTTPError, err:
        if err.code == 404:
            print "File not found (HTTP error 404)!"
        else:
            print "HTTP error (%s)" % unicode(err)
        return False
Example #23
0
    def displayProgress(self):
        """
        Display progress (to stdout) of the whole process.
        Compute data rate (in byte per sec) and time estimation.
        """
        # Program next update
        self.next_progress = time() + PROGRESS_UPDATE

        # Progress offset
        percent = float(self.current_offset - self.start_offset) * 100 / (
            self.size - self.start_offset)
        offset = self.current_offset // 8
        message = "Search: %.2f%% -- offset=%u (%s)" % (percent, offset,
                                                        humanFilesize(offset))

        # Compute data rate (byte/sec)
        average = self.datarate.average
        if average:
            message += " -- %s/sec " % humanFilesize(average // 8)
            eta = float(self.size - self.current_offset) / average
            message += " -- ETA: %s" % humanDuration(eta * 1000)

        # Display message
        print >> stderr, message
 def createDescription(self):
     desc = "Inode %s: " % self.uniq_id
     size = self["size"].value
     if self["blocks"].value == 0:
         desc += "(unused)"
     elif 11 <= self.uniq_id:
         size = humanFilesize(size)
         desc += "file, size=%s, mode=%s" % (size, self.getMode())
     else:
         if self.uniq_id in self.inode_type_name:
             desc += self.inode_type_name[self.uniq_id]
             if self.uniq_id == 2:
                 desc += " (%s)" % self.getMode()
         else:
             desc += "special"
     return desc
Example #25
0
 def createDescription(self):
     desc = "Inode %s: " % self.uniq_id
     size = self["size"].value
     if self["blocks"].value == 0:
         desc += "(unused)"
     elif 11 <= self.uniq_id:
         size = humanFilesize(size)
         desc += "file, size=%s, mode=%s" % (size, self.getMode())
     else:
         if self.uniq_id in self.inode_type_name:
             desc += self.inode_type_name[self.uniq_id]
             if self.uniq_id == 2:
                 desc += " (%s)" % self.getMode()
         else:
             desc += "special"
     return desc
    def checkPattern(self):
        if not (config.check_padding_pattern):
            return False
        if self.pattern is None:
            return False

        if self.MAX_SIZE < self._size / 8:
            self.info("only check first %s of padding" % humanFilesize(self.MAX_SIZE))
            content = self._parent.stream.readBytes(self.absolute_address, self.MAX_SIZE)
        else:
            content = self.value
        index = 0
        pattern_len = len(self.pattern)
        while index < len(content):
            if content[index : index + pattern_len] != self.pattern:
                self.warning("padding contents doesn't look normal" " (invalid pattern at byte %u)!" % index)
                return False
            index += pattern_len
        return True
Example #27
0
 def __init__(self, parent, name, length, description=None,
 parser=None, filename=None, mime_type=None, parser_class=None):
     if filename:
         if not isinstance(filename, unicode):
             filename = makePrintable(filename, "ISO-8859-1")
         if not description:
             description = 'File "%s" (%s)' % (filename, humanFilesize(length))
     Bytes.__init__(self, parent, name, length, description)
     def createInputStream(cis, **args):
         tags = args.setdefault("tags",[])
         if parser_class:
             tags.append(( "class", parser_class ))
         if parser is not None:
             tags.append(( "id", parser.PARSER_TAGS["id"] ))
         if mime_type:
             tags.append(( "mime", mime_type ))
         if filename:
             tags.append(( "filename", filename ))
         return cis(**args)
     self.setSubIStream(createInputStream)
Example #28
0
 def __init__(self, parent, name, description=None):
     FieldSet.__init__(self, parent, name, description)
     self._size = (self["size"].value + 3*4) * 8
     if MAX_CHUNK_SIZE < (self._size//8):
         raise ParserError("PNG: Chunk is too big (%s)"
             % humanFilesize(self._size//8))
     tag = self["tag"].value
     self.desc_func = None
     self.value_func = None
     if tag in self.TAG_INFO:
         self._name, self.parse_func, desc, value_func = self.TAG_INFO[tag]
         if value_func:
             self.value_func = value_func
             self.createValue = self.createValueFunc
         if desc:
             if isinstance(desc, str):
                 self._description = desc
             else:
                 self.desc_func = desc
     else:
         self._description = ""
         self.parse_func = None
Example #29
0
    def checkPattern(self):
        if not (config.check_padding_pattern):
            return False
        if self.pattern is None:
            return False

        if self.MAX_SIZE < self._size / 8:
            self.info("only check first %s of padding" %
                      humanFilesize(self.MAX_SIZE))
            content = self._parent.stream.readBytes(self.absolute_address,
                                                    self.MAX_SIZE)
        else:
            content = self.value
        index = 0
        pattern_len = len(self.pattern)
        while index < len(content):
            if content[index:index + pattern_len] != self.pattern:
                self.warning("padding contents doesn't look normal"
                             " (invalid pattern at byte %u)!" % index)
                return False
            index += pattern_len
        return True
Example #30
0
    def processParser(self, offset, parser):
        """
        Process a valid parser.
        """
        text = "[+] File at %s" % (offset // 8)
        if parser.content_size is not None:
            text += " size=%s (%s)" % (parser.content_size // 8, humanFilesize(parser.content_size // 8))
        if not (parser.content_size) or parser.content_size // 8 < FILE_MAX_SIZE:
            text += ": " + parser.description
        else:
            text += ": " + parser.__class__.__name__

        if self.output and parser.content_size:
            if offset == 0 and parser.content_size == self.size:
                text += " (don't copy whole file)"
            elif parser.content_size // 8 >= FILE_MAX_SIZE:
                text += " (don't copy file, too big)"
            elif not self.filter or self.filter(parser):
                filename = self.output.createFilename(parser.filename_suffix)
                filename = self.output.writeFile(filename, self.stream, offset, parser.content_size)
                text += " => %s" % filename
        print text
        self.next_progress = time() + PROGRESS_UPDATE
 def createDescription(self):
     desc = "Group %s: %s" % (self.uniq_id, humanFilesize(self.size / 8))
     if "superblock_copy" in self:
         desc += " (with superblock copy)"
     return desc
Example #32
0
 def createDescription(self):
     desc = "Group %s: %s" % (self.uniq_id, humanFilesize(self.size/8))
     if "superblock_copy" in self:
         desc += " (with superblock copy)"
     return desc
Example #33
0
 def createDisplay(self):
     if not self["value"].hasValue():
         return None
     if self._name in ("length", "piece_length"):
         return humanFilesize(self.value)
     return FieldSet.createDisplay(self)
Example #34
0
 def createDescription(self):
     size = self["nb_sectors"].value * self["bios/bytes_per_sector"].value
     return "NTFS Master Boot Record (%s)" % humanFilesize(size)
Example #35
0
    def update(self, node):
        if node.depth:
            text = ' ' * (3 * node.depth - 2)
            if node.childs:
                text += '- '
            elif node.field.is_field_set:
                text += '+ '
            else:
                text += '  '
            name = node.field.name
        else:
            text = ''
            name = node.field.stream.source

        if node.field.size:
            if self.flags & self.use_absolute_address:
                address = node.field.absolute_address
            else:
                address = node.field.address
            display_bits = (address % 8) != 0 or (node.field.size % 8) != 0

            if self.flags & self.hex_address:
                if display_bits:
                    text += "%04x.%x" % (address/8, address%8)
                else:
                    text += "%04x" % (address/8)
            else:
                if display_bits:
                    text += "%u.%u" % (address/8, address%8)
                else:
                    text += "%u" % (address/8)
            text += ") " + name
        else:
            text += "-> " + name

        smart_display = True
        if self.flags & self.display_value and node.field.hasValue():
            if self.flags & self.human_size:
                display = node.field.display
            else:
                display = node.field.raw_display
                smart_display = False
            text += "= %s" % display
        if node.field.description and self.flags & self.display_description:
            description = node.field.description
            if not(self.flags & self.human_size):
                description = makePrintable(description, "ASCII")
            text += ": %s" % description
        if self.flags & self.display_size and node.field.size or self.flags & self.display_type:
            tmp_text = []
            if self.flags & self.display_type:
                tmp_text.append(node.field.getFieldType())
            if self.flags & self.display_size:
                if node.field.size % 8:
                    tmp_text.append( humanBitSize(node.field.size) )
                else:
                    size = node.field.size / 8
                    if not self.flags & self.human_size:
                        tmp_text.append( ngettext("%u byte", "%u bytes", size) % size)
                    else:
                        tmp_text.append( humanFilesize(size) )
            text += " (%s)" % ", ".join(tmp_text)
        text = makePrintable(text, self.charset, to_unicode=True, smart=smart_display)
        node.setText(text, self.flags)
Example #36
0
    def update(self, node):
        if node.depth:
            text = ' ' * (3 * node.depth - 2)
            if node.childs:
                text += '- '
            elif node.field.is_field_set:
                text += '+ '
            else:
                text += '  '
            name = node.field.name
        else:
            text = ''
            name = node.field.stream.source

        if node.field.size:
            if self.flags & self.use_absolute_address:
                address = node.field.absolute_address
            else:
                address = node.field.address
            display_bits = (address % 8) != 0 or (node.field.size % 8) != 0

            if self.flags & self.hex_address:
                if display_bits:
                    text += "%04x.%x" % (address/8, address%8)
                else:
                    text += "%04x" % (address/8)
            else:
                if display_bits:
                    text += "%u.%u" % (address/8, address%8)
                else:
                    text += "%u" % (address/8)
            text += ") " + name
        else:
            text += "-> " + name

        smart_display = True
        if self.flags & self.display_value and node.field.hasValue():
            if self.flags & self.human_size:
                display = node.field.display
            else:
                display = node.field.raw_display
                smart_display = False
            text += "= %s" % display
        if node.field.description and self.flags & self.display_description:
            text += ": %s" % node.field.description
        if self.flags & self.display_size and node.field.size or self.flags & self.display_type:
            tmp_text = []
            if self.flags & self.display_type:
                tmp_text.append(node.field.getFieldType())
            if self.flags & self.display_size:
                if node.field.size % 8:
                    tmp_text.append( humanBitSize(node.field.size) )
                else:
                    size = node.field.size / 8
                    if not self.flags & self.human_size:
                        tmp_text.append( ngettext("%u byte", "%u bytes", size) % size)
                    else:
                        tmp_text.append( humanFilesize(size) )
            text += " (%s)" % ", ".join(tmp_text)
        text = makePrintable(text, self.charset, to_unicode=True, smart=smart_display)
        node.setText(text, self.flags)
Example #37
0
 def createDisplay(self):
     if not self["value"].hasValue():
         return None
     if self._name in ("length", "piece_length"):
         return humanFilesize(self.value)
     return FieldSet.createDisplay(self)
Example #38
0
 def createDescription(self):
     return "%d blocks of %s" % (self["num_blocks"].value,
                                 humanFilesize(self["len"].value))
Example #39
0
 def createDescription(self):
     return "%d blocks of %s" % (
         self["num_blocks"].value, humanFilesize(self["len"].value))
Example #40
0
 def createDescription(self):
     size = self["nb_sectors"].value * self["bios/bytes_per_sector"].value
     return "NTFS Master Boot Record (%s)" % humanFilesize(size)