def _read_directory(self):
if self.is_ncf():
# Make NCF files "readable" by a configurable folder much like Steam's "common" folder
path = self.ncf_folder_pattern.replace("/", os.sep)
if ("{NAME}" in path
or "{FILE}" in path) and not hasattr(self, "filename"):
raise ValueError, "NCF folder path has {NAME} or {FILE} but filename couldn't be figured out. Please set manually."
path = path.replace("{NAME}",
".".join(self.filename.split(".")[:-1]))
path = path.replace("{FILE}", self.filename)
package = FilesystemPackage()
package.dot_replacement = self.dot_replacement
package.parse(path)
elif self.is_gcf():
package = self
manifest_entry = self.manifest.manifest_entries[0]
# Fill in root.
self.root = DirectoryFolder(self)
self.root.index = 0
self.root.package = package
self.root.name = ""
self.root._manifest_entry = manifest_entry
self._read_directory_table(self.root)
print "Building Split Map"
self.root.build_split_map()
def _read_directory_table(self, folder):
i = folder._manifest_entry.child_index
while i != sys.maxint and i != 0:
manifest_entry = self.manifest.manifest_entries[i]
is_file = manifest_entry.directory_flags & CacheFileManifestEntry.FLAG_IS_FILE != 0
# Create our entry.
if is_file:
entry = DirectoryFile(folder)
else:
entry = DirectoryFolder(folder)
entry.package = self
entry._manifest_entry = manifest_entry
entry.item_size = manifest_entry.item_size
entry.index = manifest_entry.index
entry.name = manifest_entry.name
folder.items[entry.name] = entry
if is_file:
# Make sure it's a GCF before we read.
if self.is_gcf():
self._read_file_table(entry)
else:
self._read_directory_table(entry)
i = manifest_entry.next_index
def _read_directory(self):
if self.is_ncf():
# Make NCF files "readable" by a configurable folder much like Steam's "common" folder
path = self.ncf_folder_pattern.replace("/", os.sep)
if ("{NAME}" in path or "{FILE}" in path) and not hasattr(self, "filename"):
raise ValueError, "NCF folder path has {NAME} or {FILE} but filename couldn't be figured out. Please set manually."
path = path.replace("{NAME}", ".".join(self.filename.split(".")[:-1]))
path = path.replace("{FILE}", self.filename)
package = FilesystemPackage()
package.dot_replacement = self.dot_replacement
package.parse(path)
elif self.is_gcf():
package = self
manifest_entry = self.manifest.manifest_entries[0]
# Fill in root.
self.root = DirectoryFolder(self)
self.root.index = 0
self.root.package = package
self.root.name = ""
self.root._manifest_entry = manifest_entry
self._read_directory_table(self.root)
print "Building Split Map"
self.root.build_split_map()
class CacheFile(object):
# Constructor
def __init__(self):
self.is_parsed = False
self.blocks = None
self.alloc_table = None
self.manifest = None
self.checksum_map = None
self.data_header = None
self.complete_total = 0
self.complete_available = 0
self.dot_replacement = "."
self.ncf_folder_pattern = "common/{NAME}"
def __del__(self):
del self.is_parsed
del self.blocks
del self.alloc_table
del self.manifest
del self.checksum_map
del self.data_header
# Main methods.
@classmethod
def parse(cls, stream):
self = cls()
try:
self.filename = os.path.split(os.path.realpath(stream.name))[1]
except AttributeError:
pass
print "Parsing"
# Header
self.stream = stream
self.header = CacheFileHeader(self)
self.header.parse(stream.read(44))
self.header.validate()
if self.is_gcf():
# Block Entries
self.blocks = CacheFileBlockAllocationTable(self)
self.blocks.parse(stream)
self.blocks.validate()
# Allocation Table
self.alloc_table = CacheFileAllocationTable(self)
self.alloc_table.parse(stream)
self.alloc_table.validate()
# Manifest
self.manifest = CacheFileManifest(self)
self.manifest.parse(stream)
self.manifest.validate()
# Checksum Map
self.checksum_map = CacheFileChecksumMap(self)
self.checksum_map.parse(stream)
self.checksum_map.validate()
if self.is_gcf():
# Data Header.
self.data_header = CacheFileSectorHeader(self)
self.data_header.parse(
stream.read(24)) # size of BlockDataHeader (6 longs)
self.data_header.validate()
print "Reading Directory Table"
self.is_parsed = True
self._read_directory()
return self
## def serialize(self):
## stream = StringIO()
## self.header.validate()
## stream.write(self.header.serialize())
## if self.is_gcf():
## self.blocks.validate()
## stream.write(self.blocks.serialize())
## self.alloc_table.validate()
## stream.write(self.alloc_table.serialize())
## self.manifest.validate()
## stream.write(self.manifest.serialize())
## self.checksum_map.validate()
## stream.write(self.checksum_map.serialize())
## if self.is_gcf():
## self.data_header.validate()
## stream.write(self.data_header.serialize())
## stream.seek(self.data_header.first_sector_offset, os.SEEK_SET)
## for data in self.sector_data:
## stream.write(data)
## return stream
# Private Methods
def _read_directory(self):
if self.is_ncf():
# Make NCF files "readable" by a configurable folder much like Steam's "common" folder
path = self.ncf_folder_pattern.replace("/", os.sep)
if ("{NAME}" in path
or "{FILE}" in path) and not hasattr(self, "filename"):
raise ValueError, "NCF folder path has {NAME} or {FILE} but filename couldn't be figured out. Please set manually."
path = path.replace("{NAME}",
".".join(self.filename.split(".")[:-1]))
path = path.replace("{FILE}", self.filename)
package = FilesystemPackage()
package.dot_replacement = self.dot_replacement
package.parse(path)
elif self.is_gcf():
package = self
manifest_entry = self.manifest.manifest_entries[0]
# Fill in root.
self.root = DirectoryFolder(self)
self.root.index = 0
self.root.package = package
self.root.name = ""
self.root._manifest_entry = manifest_entry
self._read_directory_table(self.root)
print "Building Split Map"
self.root.build_split_map()
def _read_directory_table(self, folder):
i = folder._manifest_entry.child_index
while i != sys.maxint and i != 0:
manifest_entry = self.manifest.manifest_entries[i]
is_file = manifest_entry.directory_flags & CacheFileManifestEntry.FLAG_IS_FILE != 0
# Create our entry.
if is_file:
entry = DirectoryFile(folder)
else:
entry = DirectoryFolder(folder)
entry.package = self
entry._manifest_entry = manifest_entry
entry.item_size = manifest_entry.item_size
entry.index = manifest_entry.index
entry.name = manifest_entry.name
folder.items[entry.name] = entry
if is_file:
# Make sure it's a GCF before we read.
if self.is_gcf():
self._read_file_table(entry)
else:
self._read_directory_table(entry)
i = manifest_entry.next_index
@raise_ncf_error
def _read_file_table(self, entry):
# Flags
# entry.flags = self.blocks[entry.index].entry_flags
# Entries of sectors
entry.sectors = []
# Number of blocks in this entry.
entry.num_of_blocks = ceil(
float(entry.size()) / float(self.data_header.sector_size))
for block in entry._manifest_entry.blocks:
if block is None:
entry.sectors = []
entry.is_fragmented = False
else:
# Sector
entry.is_fragmented = block.is_fragmented
entry.sectors = block.sectors
self.complete_available += block.file_data_size
entry.is_user_config = entry.index in self.manifest.user_config_entries
entry.is_minimum_footprint = entry.index in self.manifest.minimum_footprint_entries
@raise_ncf_error
def _merge_file_blocks(self, entry):
terminator = 0xFFFFFFFF if self.alloc_table.is_long_terminator == 1 else 0xFFFF
deleted_blocks = []
# If we are in one block, return plz.
if not entry.first_block.next_block is not None:
return
# Go through the blocks of each file.
for block in entry.blocks:
# Get our first sector.
sector_index = block.first_sector_index
# From that, find the last sector in the block.
while self.alloc_table[sector_index] != terminator:
sector_index = self.alloc_table[sector_index]
# Set the link from the last sector in the previous block to the first sector in this block.
self.alloc_table[sector_index] = block.first_sector_index
# Set the block to be deleted later.
deleted_blocks.append(block)
# Delete the block.
for block in deleted_blocks:
del block
# Internal methods.
def _path_sep(self):
return STEAM_TERMINATOR
def _join_path(self, *args):
return STEAM_TERMINATOR.join(args)
def _split_path(self, name):
return name.split(STEAM_TERMINATOR)
@raise_parse_error
@raise_ncf_error
def _size(self, file):
return self.manifest.manifest_entries[file.index].item_size
@raise_parse_error
@raise_ncf_error
def _open_file(self, file, mode):
return GCFFileStream(file, self, mode)
@raise_parse_error
@raise_ncf_error
def _extract_folder(self,
folder,
where,
recursive,
keep_folder_structure,
item_filter=None):
if keep_folder_structure:
try:
os.makedirs(os.path.join(where, folder.sys_path()))
except Exception:
pass
# Loop over the folder and extract files and folders (if recursive)
for key, entry in folder.items_nomagic.iteritems():
# Don't bother recursing (and creating the folder) if no files are left after the filter.
if entry.is_folder() and recursive and (
(item_filter is None) or
(len([x for x in entry.all_files() if item_filter(x)]) > 0)):
self._extract_folder(entry, where, True, keep_folder_structure,
item_filter)
elif entry.is_file():
if (item_filter is None) or item_filter(entry):
self._extract_file(entry, where, keep_folder_structure)
@raise_parse_error
@raise_ncf_error
def _extract_file(self, file, where, keep_folder_structure):
global MAX_FILENAME
output = "\rExtracting %r..." % (file.sys_path(), )
MAX_FILENAME = max(len(output), MAX_FILENAME)
print output, " " * (MAX_FILENAME - len(output)),
if keep_folder_structure:
fsHandle = open(os.path.join(where, file.sys_path()), "wb")
else:
fsHandle = open(os.path.join(where, file.name), "wb")
cacheStream = self._open_file(file, "rb")
fsHandle.write(cacheStream.readall())
cacheStream.close()
fsHandle.close()
# Public Methods
def is_ncf(self):
return self.header.is_ncf()
def is_gcf(self):
return self.header.is_gcf()
@raise_parse_error
@raise_ncf_error
def complete_percent(self, range=100):
return float(self.complete_available) / float(
self.complete_total) * float(range)
@raise_parse_error
@raise_ncf_error
def extract(self,
where,
recursive=True,
keep_folder_structure=True,
filter=None):
self._extract_folder(self.root, where, recursive,
keep_folder_structure, filter)
@raise_parse_error
@raise_ncf_error
def extract_minimum_footprint(self, where, keep_folder_structure=True):
self._extract_folder(
self.root, where, True, keep_folder_structure,
lambda x: x.is_minimum_footprint and not (os.path.exists(
os.path.join(where, x.sys_path())) and x.is_user_config))
def open(self, filename, mode):
# Use file.open instead of _open_file as we may be parsing an NCF
return self.root.find_item(filename).open(mode)
# Magic/Special Methods
@raise_parse_error
def __len__(self):
return len(self.root)
@raise_parse_error
def __iter__(self):
# for i in gcf:
return self.root.__iter__()
@raise_parse_error
def __getattr__(self, name):
# gcf.folder1.folder2.file.txt
return self.root.__getattr__(name)
@raise_parse_error
def __getitem__(self, name):
# gcf["folder1"]["folder2"]["file.txt"]
return self.root.__getitem__(name)
class CacheFile(object):
# Constructor
def __init__(self):
self.is_parsed = False
self.blocks = None
self.alloc_table = None
self.manifest = None
self.checksum_map = None
self.data_header = None
self.complete_total = 0
self.complete_available = 0
self.dot_replacement = "."
self.ncf_folder_pattern = "common/{NAME}"
def __del__(self):
del self.is_parsed
del self.blocks
del self.alloc_table
del self.manifest
del self.checksum_map
del self.data_header
# Main methods.
@classmethod
def parse(cls, stream):
self = cls()
try:
self.filename = os.path.split(os.path.realpath(stream.name))[1]
except AttributeError:
pass
print "Parsing"
# Header
self.stream = stream
self.header = CacheFileHeader(self)
self.header.parse(stream.read(44))
self.header.validate()
if self.is_gcf():
# Block Entries
self.blocks = CacheFileBlockAllocationTable(self)
self.blocks.parse(stream)
self.blocks.validate()
# Allocation Table
self.alloc_table = CacheFileAllocationTable(self)
self.alloc_table.parse(stream)
self.alloc_table.validate()
# Manifest
self.manifest = CacheFileManifest(self)
self.manifest.parse(stream)
self.manifest.validate()
# Checksum Map
self.checksum_map = CacheFileChecksumMap(self)
self.checksum_map.parse(stream)
self.checksum_map.validate()
if self.is_gcf():
# Data Header.
self.data_header = CacheFileSectorHeader(self)
self.data_header.parse(stream.read(24)) # size of BlockDataHeader (6 longs)
self.data_header.validate()
print "Reading Directory Table"
self.is_parsed = True
self._read_directory()
return self
## def serialize(self):
## stream = StringIO()
## self.header.validate()
## stream.write(self.header.serialize())
## if self.is_gcf():
## self.blocks.validate()
## stream.write(self.blocks.serialize())
## self.alloc_table.validate()
## stream.write(self.alloc_table.serialize())
## self.manifest.validate()
## stream.write(self.manifest.serialize())
## self.checksum_map.validate()
## stream.write(self.checksum_map.serialize())
## if self.is_gcf():
## self.data_header.validate()
## stream.write(self.data_header.serialize())
## stream.seek(self.data_header.first_sector_offset, os.SEEK_SET)
## for data in self.sector_data:
## stream.write(data)
## return stream
# Private Methods
def _read_directory(self):
if self.is_ncf():
# Make NCF files "readable" by a configurable folder much like Steam's "common" folder
path = self.ncf_folder_pattern.replace("/", os.sep)
if ("{NAME}" in path or "{FILE}" in path) and not hasattr(self, "filename"):
raise ValueError, "NCF folder path has {NAME} or {FILE} but filename couldn't be figured out. Please set manually."
path = path.replace("{NAME}", ".".join(self.filename.split(".")[:-1]))
path = path.replace("{FILE}", self.filename)
package = FilesystemPackage()
package.dot_replacement = self.dot_replacement
package.parse(path)
elif self.is_gcf():
package = self
manifest_entry = self.manifest.manifest_entries[0]
# Fill in root.
self.root = DirectoryFolder(self)
self.root.index = 0
self.root.package = package
self.root.name = ""
self.root._manifest_entry = manifest_entry
self._read_directory_table(self.root)
print "Building Split Map"
self.root.build_split_map()
def _read_directory_table(self, folder):
i = folder._manifest_entry.child_index
while i != sys.maxint and i != 0:
manifest_entry = self.manifest.manifest_entries[i]
is_file = manifest_entry.directory_flags & CacheFileManifestEntry.FLAG_IS_FILE != 0
# Create our entry.
if is_file:
entry = DirectoryFile(folder)
else:
entry = DirectoryFolder(folder)
entry.package = self
entry._manifest_entry = manifest_entry
entry.item_size = manifest_entry.item_size
entry.index = manifest_entry.index
entry.name = manifest_entry.name
folder.items[entry.name] = entry
if is_file:
# Make sure it's a GCF before we read.
if self.is_gcf():
self._read_file_table(entry)
else:
self._read_directory_table(entry)
i = manifest_entry.next_index
@raise_ncf_error
def _read_file_table(self, entry):
# Flags
# entry.flags = self.blocks[entry.index].entry_flags
# Entries of sectors
entry.sectors = []
# Number of blocks in this entry.
entry.num_of_blocks = ceil(float(entry.size()) / float(self.data_header.sector_size))
for block in entry._manifest_entry.blocks:
if block is None:
entry.sectors = []
entry.is_fragmented = False
else:
# Sector
entry.is_fragmented = block.is_fragmented
entry.sectors = block.sectors
self.complete_available += block.file_data_size
entry.is_user_config = entry.index in self.manifest.user_config_entries
entry.is_minimum_footprint = entry.index in self.manifest.minimum_footprint_entries
@raise_ncf_error
def _merge_file_blocks(self, entry):
terminator = 0xFFFFFFFF if self.alloc_table.is_long_terminator == 1 else 0xFFFF
deleted_blocks = []
# If we are in one block, return plz.
if not entry.first_block.next_block is not None:
return
# Go through the blocks of each file.
for block in entry.blocks:
# Get our first sector.
sector_index = block.first_sector_index
# From that, find the last sector in the block.
while self.alloc_table[sector_index] != terminator:
sector_index = self.alloc_table[sector_index]
# Set the link from the last sector in the previous block to the first sector in this block.
self.alloc_table[sector_index] = block.first_sector_index
# Set the block to be deleted later.
deleted_blocks.append(block)
# Delete the block.
for block in deleted_blocks:
del block
# Internal methods.
def _path_sep(self):
return STEAM_TERMINATOR
def _join_path(self, *args):
return STEAM_TERMINATOR.join(args)
def _split_path(self, name):
return name.split(STEAM_TERMINATOR)
@raise_parse_error
@raise_ncf_error
def _size(self, file):
return self.manifest.manifest_entries[file.index].item_size
@raise_parse_error
@raise_ncf_error
def _open_file(self, file, mode):
return GCFFileStream(file, self, mode)
@raise_parse_error
@raise_ncf_error
def _extract_folder(self, folder, where, recursive, keep_folder_structure, item_filter=None):
if keep_folder_structure:
try:
os.makedirs(os.path.join(where, folder.sys_path()))
except Exception:
pass
# Loop over the folder and extract files and folders (if recursive)
for key, entry in folder.items_nomagic.iteritems():
# Don't bother recursing (and creating the folder) if no files are left after the filter.
if entry.is_folder() and recursive and ((item_filter is None) or (len([x for x in entry.all_files() if item_filter(x)]) > 0)):
self._extract_folder(entry, where, True, keep_folder_structure, item_filter)
elif entry.is_file():
if (item_filter is None) or item_filter(entry):
self._extract_file(entry, where, keep_folder_structure)
@raise_parse_error
@raise_ncf_error
def _extract_file(self, file, where, keep_folder_structure):
global MAX_FILENAME
output = "\rExtracting %r..." % (file.sys_path(),)
MAX_FILENAME = max(len(output), MAX_FILENAME)
print output, " "*(MAX_FILENAME-len(output)),
if keep_folder_structure:
fsHandle = open(os.path.join(where, file.sys_path()), "wb")
else:
fsHandle = open(os.path.join(where, file.name), "wb")
cacheStream = self._open_file(file, "rb")
fsHandle.write(cacheStream.readall())
cacheStream.close()
fsHandle.close()
# Public Methods
def is_ncf(self):
return self.header.is_ncf()
def is_gcf(self):
return self.header.is_gcf()
@raise_parse_error
@raise_ncf_error
def complete_percent(self, range=100):
return float(self.complete_available) / float(self.complete_total) * float(range)
@raise_parse_error
@raise_ncf_error
def extract(self, where, recursive=True, keep_folder_structure=True, filter=None):
self._extract_folder(self.root, where, recursive, keep_folder_structure, filter)
@raise_parse_error
@raise_ncf_error
def extract_minimum_footprint(self, where, keep_folder_structure=True):
self._extract_folder(self.root, where, True, keep_folder_structure, lambda x:x.is_minimum_footprint and not (os.path.exists(os.path.join(where, x.sys_path())) and x.is_user_config))
def open(self, filename, mode):
# Use file.open instead of _open_file as we may be parsing an NCF
return self.root.find_item(filename).open(mode)
# Magic/Special Methods
@raise_parse_error
def __len__(self):
return len(self.root)
@raise_parse_error
def __iter__(self):
# for i in gcf:
return self.root.__iter__()
@raise_parse_error
def __getattr__(self, name):
# gcf.folder1.folder2.file.txt
return self.root.__getattr__(name)
@raise_parse_error
def __getitem__(self, name):
# gcf["folder1"]["folder2"]["file.txt"]
return self.root.__getitem__(name)
