for y in ('Skyrim.esm', # Skyrim

'Update.esm',

'Dawnguard.esm',

'Hearthfires.esm',

'Dragonborn.esm',

'Fallout3.esm', # Fallout 3

'Anchorage.esm',

'BrokenSteel.esm',

'PointLookout.esm',

'ThePitt.esm',

'Zeta.esm',

'FalloutNV.esm', # Fallout New Vegas

'CaravanPack.esm',

'ClassicPack.esm',

'DeadMoney.esm',

'GunRunnersArsenal.esm',

'HonestHearts.esm',

'LonesomeRoad.esm',

'MercenaryPack.esm',

'OldWorldBlues.esm',

'TribalPack.esm',

'Oblivion.esm', # Oblivion

# 'DLCShiveringIsles.esp, # dummy plugin

'DLCFrostcrag.esp',

'DLCBattlehornCastle.esp',

'DLCSpellTomes.esp',

'DLCMehrunesRazor.esp',

'DLCOrrery.esp',

'DLCThievesDen.esp',

'DLCHorseArmor.esp',

'DLCVileLair.esp',

'Knights.esp',

)

# Oblivion

True: {'headerSize': 20,

},

# non-Oblivion

False: {'headerSize': 24,

},

"""File-object with convenience reading functions."""

def unpack(self, fmt):

return struct.unpack(fmt, self.read(struct.calcsize(fmt)))

def readUByte(self): return struct.unpack('B', self.read(1))[0]

def readUInt16(self): return struct.unpack('H', self.read(2))[0]

def readUInt32(self): return struct.unpack('I', self.read(4))[0]

def readUInt64(self): return struct.unpack('Q', self.read(8))[0]

def readByte(self): return struct.unpack('b', self.read(1))[0]

def readInt16(self): return struct.unpack('h', self.read(2))[0]

def readInt32(self): return struct.unpack('i', self.read(4))[0]

"""Simple console-based progress bar."""

__slots__ = ('prefix', 'end', 'cur', 'length', 'outFile', 'percent',)

def __init__(self, prefix='', maxValue=100, length=20, percent=True,

padPrefix=None, file=sys.stdout):

if padPrefix:

self.prefix = ('{:<%s}' % padPrefix).format(prefix)

else:

self.prefix = prefix

# Make sure it'll fit in the console

try:

maxWidth = shutil.get_terminal_size()[0]

except:

maxWidth = 80

# Calculate length of current message

# +9 accounts for spacing, brackets, percentage, and one empty

# space to prevent scrolling to the next line automatically

width = len(self.prefix) + length + 9

if width > maxWidth:

extra = width - maxWidth

# Too long, make things smaller

if length > 20:

remove = length - max(20, length - extra)

extra -= remove

length -= remove

if extra > 0:

# Still too much

remove = self.prefix[-extra:]

self.prefix = self.prefix[:-extra]

if remove != ' '*extra:

# Had to remove some text

self.prefix = self.prefix[:-3] + '...'

self.end = maxValue

self.length = length

self.cur = 0

self.outFile = file

self.percent = percent

def update(self):

"""Increment progress by 1."""

self(self.cur+1)

def __call__(self, pos):

"""Set progress bar to specified amount."""

self.cur = max(0, min(self.end, pos))

# Build progress bar:

percent = self.cur / self.end

filled = percent * self.length

partial = filled - int(filled)

filled = int(filled)

empty = self.length - filled - 1

bar = '#' * filled

if self.cur == 0:

bar += ' '

elif self.cur == self.end:

pass

elif partial < 0.25:

bar += ' '

elif partial < 0.50:

bar += '-'

elif partial < 0.75:

bar += '+'

else:

bar += '*'

bar += ' ' * empty

# Print

msg = '\r%s [%s]' % (self.prefix, bar)

if self.percent:

msg += '{:>4}%'.format(int(percent * 100))

print(msg, end='', file=self.outFile, flush=True)

def fill(self):

self(self.end)

print('', file=self.outFile)

def __enter__(self):

return self

def __exit__(self, exc_type, exc_value, traceback):

if exc_type is None:

self.fill()

else:

"""Main function, fires everything off."""

opts = parser.parse_args()

if opts.all:

# Dump stats for every plugin

to_dump = [x.lower() for x in os.listdir() if rePlugin.search(x)]

to_dump.sort()

# Move Skryim.esm/Update.esm first

for plugin in reversed(officialPlugins):

if plugin in to_dump:

i = to_dump.index(plugin)

del to_dump[i]

to_dump.insert(0, plugin)

elif opts.plugin:

# Dump specified plugin

plugin = opts.plugin.lower()

if plugin.endswith('.ghost'):

plugin = plugin[-6:]

to_dump = [x for x in (plugin, plugin+'.ghost')

if os.path.exists(x)]

else:

# Only dump stats for offical plugins

to_dump = [x for x in officialPlugins if os.path.exists(x)]

if not to_dump:

print('Could not find any plugins to dump. Are you sure TESDumpStats'

' is in the Skyrim Data directory?')

return

# Check to see if any plugins also have their ghosted version present.

# We'll dump both if they exist, but something wonky could be up with the

# user's Data directory.

dupes = []

for plugin in to_dump:

if plugin+'.ghost' in to_dump:

dupes.append(plugin)

if dupes:

print('WARNING: The following plugins exist in your Data directory as'

' both plugins and ghosted plugins. Something may be wrong! '

'The plugins will both be processed however.')

for dupe in dupes:

print(' ', dupe)

# Setup output directory/file

timestamp = time.strftime('%Y-%m-%d_%H%M.%S')

outDir = (opts.output if opts.output else

os.path.join(os.getcwd(),'TESDumpStats'))

if opts.split:

outDir = os.path.join(outDir, timestamp)

try:

if not os.path.exists(outDir):

os.makedirs(outDir)

testFile = os.path.join(outDir,'test.txt')

with open(testFile,'wb'):

pass

os.remove(testFile)

except Exception as e:

print('ERROR: Could not setup output path specified:\n\n'

' ' + outDir + '\n\n'

' ' + str(e) + '\n\n')

return

# Start dumping

print('Beginning dump.')

print('Output directory:', outDir)

print('')

stats = dict()

padLength = max([len(x) for x in to_dump])

try:

for plugin in to_dump:

s = stats.setdefault(plugin, dict())

dumpPlugin(plugin, s, padLength, opts.oblivion)

print('Writing statistics...')

printStats(stats, outDir, opts)

print('Dump complete.')

except KeyboardInterrupt:

"""Gets stats about records, etc from fileName, updates stats dict,

then prints results to outFile."""

s = dict()

# GRUP/plugin header size

headerSize = gameFormat[oblivion]['headerSize']

# Get basic stats on the file

stats['size'] = size = os.path.getsize(fileName)

stats['time'] = os.path.getmtime(fileName)

with Progress(fileName, size, padPrefix=padLength) as progress:

try:

with FileReader(fileName, 'rb') as ins:

# Calculate CRC32

crc = 0

while ins.tell() < size:

crc = binascii.crc32(ins.read(2097152), crc)

crc = crc & 0xFFFFFFFF

stats['crc'] = crc

ins.seek(0)

# No error checking, just assume everything is properly formed

s = stats['records'] = dict()

# Read TES4 record + GRUPs

while ins.tell() < size:

dumpGRUPOrRecord(ins, s, size, progress, oblivion, headerSize)

except KeyboardInterrupt:

raise

except Exception as e:

print('ERROR: Unhandled exception\n')

suffix = 'B'

if size > 1024*10:

size /= 1024

suffix = 'KB'

if size > 1024*10:

size /= 1024

suffix = 'MB'

if size > 1024*10:

size /= 1024

suffix = 'GB'

for Type in sorted(stats):

print('', Type, file=outFile)

recStats = stats[Type]

count = recStats['count']

print(' Count:', count, file=outFile)

sizes = recStats['sizes']

minsize = min(sizes)

maxsize = max(sizes)

compressed = recStats['compressed']

if compressed == count:

print(' All compressed', file=outFile)

elif compressed > 0:

print(' Compressed: %i / %i' % (compressed, count), file=outFile)

else:

print(' None compressed', file=outFile)

if minsize == maxsize:

print(' Size:', maxsize, file=outFile)

else:

print(' Min Size:', minsize, file=outFile)

print(' Max Size:', maxsize, file=outFile)

# Subrecords

print(' Subrecords:', file=outFile)

for subtype in sorted(recStats):

if subtype in ('count','sizes','compressed'):

continue

subStats = recStats[subtype]

subCounts = subStats['counts']

if len(subCounts) == count:

# At least 1 per record

print(' ', subtype, '- Required', file=outFile)

else:

print(' ', subtype,

'- %i / %i records' % (len(subCounts), count),

file=outFile)

maxcount = max(subCounts)

mincount = min(subCounts)

if maxcount == mincount:

print(' Count:', maxcount, file=outFile)

else:

print(' Min Count:', mincount, file=outFile)

print(' Max Count:', maxcount, file=outFile)

sizes = subStats['sizes']

maxsize = max(sizes)

minsize = min(sizes)

if maxsize == minsize:

print(' Size:', maxsize, file=outFile)

else:

print(' Min Size:', minsize, file=outFile)

print(' Max Size:', maxsize, file=outFile)

for Type in source:

if Type not in dest:

dest[Type] = source[Type]

else:

destRecStats = dest[Type]

sourceRecStats = source[Type]

# Merge

destRecStats['count'] += sourceRecStats['count']

destRecStats['sizes'].extend(sourceRecStats['sizes'])

destRecStats['compressed'] += sourceRecStats['compressed']

# Subrecords

for subType in sourceRecStats:

if subType in ('compressed','sizes','count'):

continue

if subType not in destRecStats:

destRecStats[subType] = sourceRecStats[subType]

else:

destSubStats = destRecStats[subType]

sourceSubStats = sourceRecStats[subType]

destSubStats['counts'].extend(sourceSubStats['counts'])

outName = os.path.join(outDir, time.strftime('%Y-%m-%d_%H%M.%S_dump.txt'))

if not opts.split:

# Make sure combined output file is empty

if os.path.exists(outName):

os.remove(outName)

mode = 'a+'

else:

mode = 'w'

allstats = dict()

for plugin in stats:

if opts.split:

outName = os.path.join(outDir, plugin+'.txt')

with open(outName, mode) as outFile:

print(plugin, file=outFile)

pstats = stats[plugin]

print(' File Size:', formatSize(pstats['size']), file=outFile)

print(' File Date:',

datetime.datetime.fromtimestamp(pstats['time']),

file=outFile)

print(' File CRC: 0x%X' % pstats['crc'], file=outFile)

recStats = pstats['records']

printRecordStats(recStats, outFile)

mergeRecordStats(allstats, recStats)

if len(stats) > 1:

if opts.split:

outName = os.path.join(outDir, 'combined_stats.txt')

with open(outName, mode) as outFile:

print('Combined Stats:', file=outFile)

pos = ins.tell()

progress(pos)

if pos+headerSize > end:

ins.seek(end)

return

grup = ins.read(4)

if grup == b'GRUP':

# It's a GRUP

size = ins.readUInt32() - headerSize

label = ins.read(4)

Type = ins.readInt32()

stamp = ins.readUInt16()

unk1 = ins.readUInt16()

if not oblivion:

version = ins.readUInt16()

unk2 = ins.readUInt16()

pos = ins.tell()

if pos+size > end:

ins.seek(end)

return

# Data

while ins.tell() < pos+size:

dumpGRUPOrRecord(ins, stats, pos+size, progress, oblivion, headerSize)

else:

Type = grup.decode('ascii')

dataSize = ins.readUInt32()

flags = ins.readUInt32()

id = ins.readUInt32()

revision = ins.readUInt32()

if not oblivion:

version = ins.readUInt16()

unk = ins.readUInt16()

if not flags & 0x20: # Not deleted

# Data

s = stats.setdefault(Type, dict())

num = s.get('count', 0)

s['count'] = num + 1

num = s.get('compressed', 0)

data = ins.read(dataSize)

if flags & 0x00040000:

# Data is compressed

uncompSize = struct.unpack('I', data[:4])

data = zlib.decompress(data[4:])

num += 1

s['compressed'] = num

s.setdefault('sizes',[]).append(len(data))

dumpSubRecords(data, s)

# Ensure we're at the end of the record

size = len(data)

pos = 0

counts = dict()

while pos < size - 6:

subType = data[pos:pos+4].decode('ascii')

pos += 4

if subType == 'XXXX':

subSize = struct.unpack('I', data[pos:pos+4])[0]

pos += 4

subType = data[pos:pos+4].decode('ascii')

pos += 4

pos += 2 # datasize

else:

subSize = struct.unpack('H', data[pos:pos+2])[0]

pos += 2

if pos+subSize > size:

break

pos += subSize

s = stats.setdefault(subType, dict())

num = counts.get(subType,0)

counts[subType] = num + 1

s.setdefault('sizes',[]).append(subSize)

for subType in counts: