def test_multi_keyvalue_pairs(self):
INPUT = '''
"root1"
{
"key1" "value1"
key2 "value2"
"key3" value3
}
root2
{
"key1" "value1"
key2 "value2"
"key3" value3
}
'''
EXPECTED = {
'root1': {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
},
'root2': {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
}
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def get_items_game(api_key=None, github=True):
sub_api = 'GetSchemaUrl'
version = 'v0001'
if github is False and api_key is not None:
url = api_request(sub_api, version, api_key)
if url is not None:
url = url['result']['items_game_url']
request = requests.get(url)
if request.status_code == 200:
items_game = vdf.loads(request.text)
return items_game
else:
return 'Request failed with code {}'.format(
request.status_code)
elif github is False and api_key is None:
print('Api key not provided, returning None')
return
else:
return
else:
request = requests.get(
'https://raw.githubusercontent.com/SteamDatabase/GameTracking-TF2/master/tf/scripts/items/items_game.txt'
)
if request.status_code == 200:
items_game = vdf.loads(request.text)
return items_game
else:
return 'Request failed with code {}'.format(
request.status_code)
def test_inline_opening_bracker(self):
INPUT = '''
"root1" {
"key1" {
}
key2 "value2"
"key3" value3
}
root2 { }
root3 {
"key1" "value1"
key2 {
}
"key3" value3
}
'''
EXPECTED = {
'root1': {
'key1': {},
'key2': 'value2',
'key3': 'value3',
},
'root2': {},
'root3': {
'key1': 'value1',
'key2': {},
'key3': 'value3',
}
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_parse_bom_removal(self):
result = vdf.loads(vdf.BOMS + '"asd" "123"')
self.assertEqual(result, {'asd': '123'})
if sys.version_info[0] is 2:
result = vdf.loads(vdf.BOMS_UNICODE + '"asd" "123"')
self.assertEqual(result, {'asd': '123'})
def test_multi_keyvalue_pairs(self):
INPUT = '''
"root1"
{
"key1" "value1"
key2 "value2"
"key3" value3
}
root2
{
"key1" "value1"
key2 "value2"
"key3" value3
}
'''
EXPECTED = {
'root1': {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
},
'root2': {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
}
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_parse_bom_removal(self):
result = vdf.loads(vdf.BOMS + '"asd" "123"')
self.assertEqual(result, {'asd': '123'})
if sys.version_info[0] is 2:
result = vdf.loads(vdf.BOMS_UNICODE + '"asd" "123"')
self.assertEqual(result, {'asd': '123'})
def test_space_for_unquoted(self):
INPUT = 'a b c d \n efg h i\t // j k\n'
EXPECTED = {
'a': 'b c d',
'efg': 'h i',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_deep_nesting(self):
INPUT = '''
"root"
{
node1
{
"node2"
{
NODE3
{
"node4"
{
node5
{
"node6"
{
NODE7
{
"node8"
{
"key" "value"
}
}
}
}
}
}
}
}
}
'''
EXPECTED = {
'root': {
'node1': {
'node2': {
'NODE3': {
'node4': {
'node5': {
'node6': {
'NODE7': {
'node8': {
'key': 'value'
}
}
}
}
}
}
}
}
}
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_keyvalue_open_quoted(self):
INPUT = ('"key1" "a\n' 'b\n' 'c"\n' 'key2 "a\n' 'b\n' 'c"\n')
EXPECTED = {
'key1': 'a\nb\nc',
'key2': 'a\nb\nc',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_wierd_symbols_for_unquoted(self):
INPUT = 'a asd.vdf\nb language_*lol*\nc zxc_-*.sss//'
EXPECTED = {
'a': 'asd.vdf',
'b': 'language_*lol*',
'c': 'zxc_-*.sss',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_wierd_symbols_for_unquoted(self):
INPUT = 'a asd.vdf\nb language_*lol*\nc zxc_-*.sss//'
EXPECTED = {
'a': 'asd.vdf',
'b': 'language_*lol*',
'c': 'zxc_-*.sss',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_hash_key(self):
INPUT = '#include "asd.vdf"'
EXPECTED = {'#include': 'asd.vdf'}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
INPUT = '#base asd.vdf'
EXPECTED = {'#base': 'asd.vdf'}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
def test_hash_key(self):
INPUT = '#include "asd.vdf"'
EXPECTED = {'#include': 'asd.vdf'}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
INPUT = '#base asd.vdf'
EXPECTED = {'#base': 'asd.vdf'}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_routines_mapper_passing(self, mock_parse):
vdf.load(sys.stdin, mapper=dict)
mock_parse.assert_called_with(sys.stdin, mapper=dict)
vdf.loads("", mapper=dict)
mock_parse.assert_called_with("", mapper=dict)
class CustomDict(dict):
pass
vdf.load(sys.stdin, mapper=CustomDict)
mock_parse.assert_called_with(sys.stdin, mapper=CustomDict)
vdf.loads("", mapper=CustomDict)
mock_parse.assert_called_with("", mapper=CustomDict)
def test_routines_mapper_passing(self, mock_parse):
vdf.load(sys.stdin, mapper=dict)
mock_parse.assert_called_with(sys.stdin, mapper=dict)
vdf.loads("", mapper=dict)
mock_parse.assert_called_with("", mapper=dict)
class CustomDict(dict):
pass
vdf.load(sys.stdin, mapper=CustomDict)
mock_parse.assert_called_with(sys.stdin, mapper=CustomDict)
vdf.loads("", mapper=CustomDict)
mock_parse.assert_called_with("", mapper=CustomDict)
def test_deep_nesting(self):
INPUT = '''
"root"
{
node1
{
"node2"
{
NODE3
{
"node4"
{
node5
{
"node6"
{
NODE7
{
"node8"
{
"key" "value"
}
}
}
}
}
}
}
}
}
'''
EXPECTED = {
'root': {
'node1': {
'node2': {
'NODE3': {
'node4': {
'node5': {
'node6': {
'NODE7': {
'node8': {
'key': 'value'
}}}}}}}}}
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def get_product_info(self, appids=[], packageids=[]):
try:
resp = self.steam.send_job_and_wait(MsgProto(EMsg.ClientPICSProductInfoRequest),
{
'apps': map(lambda x: {'appid': x}, appids),
'packages': map(lambda x: {'packageid': x}, packageids),
},
timeout=10
)
if not resp:
return {}
resp = proto_to_dict(resp)
for app in resp.get('apps', []):
app['appinfo'] = vdf.loads(
app.pop('buffer')[:-1].decode('utf-8', 'replace'))['appinfo']
app['sha'] = hexlify(app['sha']).decode('utf-8')
for pkg in resp.get('packages', []):
pkg['appinfo'] = vdf.binary_loads(pkg.pop('buffer')[4:])[
str(pkg['packageid'])]
pkg['sha'] = hexlify(pkg['sha']).decode('utf-8')
return resp
except Exception as e:
print('get_product_info() exception: ' + str(e))
return {}
def func(name, new_struct=False):
library_dir = Path(str(tmp_path)) / "mnt" / name
library_dir.mkdir(parents=True)
# Update libraryfolders.vdf with the new library folder
libraryfolders_config = vdf.loads(
steam_libraryfolders_path.read_text())
# Each new library adds a new entry into the config file with the
# field name that starts from 1 and increases with each new library
# folder.
# Newer Steam releases stores the library entry in a dict, while
# older releases just store the full path as the field value
library_id = len(libraryfolders_config["LibraryFolders"].keys()) - 1
if new_struct:
libraryfolders_config["LibraryFolders"][str(library_id)] = {
"path": str(library_dir),
"label": "",
"mounted": "1"
}
else:
libraryfolders_config["LibraryFolders"][str(library_id)] = \
str(library_dir)
steam_libraryfolders_path.write_text(vdf.dumps(libraryfolders_config))
return library_dir
def test_get_steam_lib_paths_duplicate_paths(self, steam_dir,
steam_library_factory):
"""
Retrive Steam library folders and ensure duplicate paths (eg.
an existing path OR a symlink that resolves to an existing path)
are removed from the returned list.
Regression test for #118
"""
library_dir = steam_library_factory("TestLibrary_A")
# Create a symlink from TestLibrary_B to TestLibrary_A
(library_dir.parent / "TestLibrary_B").symlink_to(library_dir)
# Add the duplicate library folder
vdf_data = vdf.loads(
(steam_dir / "steamapps" / "libraryfolders.vdf").read_text())
vdf_data["LibraryFolders"]["2"] = str(library_dir.parent /
"TestLibrary_B")
(steam_dir / "steamapps" / "libraryfolders.vdf").write_text(
vdf.dumps(vdf_data))
library_paths = get_steam_lib_paths(steam_dir)
# Only two paths should be returned
assert len(library_paths) == 2
assert steam_dir in library_paths
assert library_dir in library_paths
def restore_config(src, dst):
"""Restore the backup. if dst exists, rename it. The new method."""
try:
with open(src, encoding='UTF-8') as backup_file:
backedup_categories = json.loads(
backup_file.read(),
object_pairs_hook=collections.OrderedDict)
with open(dst, encoding='UTF-8') as valves_file:
content = vdf.loads(valves_file.read(),
mapper=collections.OrderedDict)
target_categories = content["UserRoamingConfigStore"][
"Software"]["Valve"]["Steam"]["Apps"]
content["UserRoamingConfigStore"]["Software"]["Valve"][
"Steam"]["Apps"] = BackupAndRestore.__insert_categories__(
target_categories, backedup_categories)
new_name = dst + " " + str(datetime.datetime.now().strftime(
"%Y-%m-%d %H;%M;%S %f")) + ".bak"
os.rename(dst, new_name)
with open(dst, "w", encoding='UTF-8') as output_file:
vdf.dump(content, output_file, pretty=True)
except UnicodeDecodeError:
raise ParseException("Can't open source file to restore")
except (FileExistsError, IOError):
raise ParseException("Can't open target file to restore")
async def get_game_library_settings_file(self):
remotestorageapp = await self._http_client.get(
"https://store.steampowered.com/account/remotestorageapp/?appid=7")
remotestorageapp_text = await remotestorageapp.text(encoding="utf-8", errors="replace")
start_index = remotestorageapp_text.find("sharedconfig.vdf")
if start_index == -1:
# Fresh user, has no sharedconfig
return []
url_start = remotestorageapp_text.find('href="', start_index)
url_end = remotestorageapp_text.find('">', url_start)
url = remotestorageapp_text[int(url_start) + len('href="'): int(url_end)]
sharedconfig = vdf.loads(await get_text(await self._http_client.get(url, allow_redirects=True)))
logger.info(f"Sharedconfig file contents {sharedconfig}")
try:
apps = sharedconfig["UserRoamingConfigStore"]["Software"]["Valve"]["Steam"]["Apps"]
except KeyError:
logger.warning('Cant read users sharedconfig, assuming no tags set')
return []
game_settings = []
for app in apps:
tags = []
if "tags" in apps[app]:
for tag in apps[app]["tags"]:
tags.append(apps[app]['tags'][tag])
hidden = True if "Hidden" in apps[app] and apps[app]['Hidden'] == '1' else False
game_settings.append({'game_id': app, 'tags': tags, 'hidden': hidden})
return game_settings
def find_current_steamid3(steam_path):
def to_steamid3(steamid64):
"""Convert a SteamID64 into the SteamID3 format"""
return int(steamid64) & 0xffffffff
loginusers_path = os.path.join(steam_path, "config", "loginusers.vdf")
try:
with open(loginusers_path, "r") as f:
content = f.read()
vdf_data = vdf.loads(content)
except IOError:
return None
users = [{
"steamid3": to_steamid3(user_id),
"account_name": user_data["AccountName"],
"timestamp": user_data.get("Timestamp", 0)
} for user_id, user_data in vdf_data["users"].items()]
# Return the user with the highest timestamp, as that's likely to be the
# currently logged-in user
if users:
user = max(users, key=lambda u: u["timestamp"])
logger.info("Currently logged-in Steam user: %s", user["account_name"])
return user["steamid3"]
return None
async def _process_package_info_response(self, body):
logger.info("Processing message PICSProductInfoResponse")
message = steammessages_clientserver_pb2.CMsgClientPICSProductInfoResponse()
message.ParseFromString(body)
apps_to_parse = []
for info in message.packages:
await self.package_info_handler(str(info.packageid))
if info.packageid == 0:
# Packageid 0 contains trash entries for every user
logger.info("Skipping packageid 0 ")
continue
package_content = vdf.binary_loads(info.buffer[4:])
for app in package_content[str(info.packageid)]['appids']:
await self.app_info_handler(mother_appid=str(info.packageid), appid=str(package_content[str(info.packageid)]['appids'][app]))
apps_to_parse.append(package_content[str(info.packageid)]['appids'][app])
for info in message.apps:
app_content = vdf.loads(info.buffer[:-1].decode('utf-8', 'replace'))
try:
if app_content['appinfo']['common']['type'].lower() == 'game':
logger.info(f"Retrieved game {app_content['appinfo']['common']['name']}")
await self.app_info_handler(appid=str(app_content['appinfo']['appid']), title=app_content['appinfo']['common']['name'], game=True)
else:
await self.app_info_handler(appid=str(app_content['appinfo']['appid']), game=False)
except KeyError:
# Unrecognized app type
await self.app_info_handler(appid=str(app_content['appinfo']['appid']), game=False)
if len(apps_to_parse) > 0:
logger.info(f"Apps to parse {apps_to_parse}, {len(apps_to_parse)} entries")
await self.get_apps_info(apps_to_parse)
def func(name,
appid,
compat_tool_name=None,
library_dir=None,
add_prefix=True,
required_tool_app=None):
if not library_dir:
steamapps_dir = steam_dir / "steamapps"
else:
steamapps_dir = library_dir / "steamapps"
install_path = steamapps_dir / "common" / name
install_path.mkdir(parents=True)
if required_tool_app:
(install_path / "toolmanifest.vdf").write_text(
vdf.dumps({
"manifest": {
"require_tool_appid": required_tool_app.appid
}
}))
(steamapps_dir / "appmanifest_{}.acf".format(appid)).write_text(
vdf.dumps({
"AppState": {
"appid": str(appid),
"name": name,
"installdir": name
}
}))
# Add Wine prefix
if add_prefix:
(steamapps_dir / "compatdata" / str(appid) /
"pfx").mkdir(parents=True)
(steamapps_dir / "compatdata" / str(appid) / "pfx.lock").touch()
# Set the preferred Proton installation for the app if provided
if compat_tool_name:
steam_config = vdf.loads(steam_config_path.read_text())
steam_config["InstallConfigStore"]["Software"]["Valve"]["Steam"][
"CompatToolMapping"][str(appid)] = {
"name": compat_tool_name,
"config": "",
"Priority": "250"
}
steam_config_path.write_text(vdf.dumps(steam_config))
steam_app = SteamApp(name=name,
appid=appid,
install_path=str(steamapps_dir / "common" / name),
prefix_path=str(steamapps_dir / "compatdata" /
str(appid) / "pfx"))
if required_tool_app:
# In actual code, `required_tool_app` attribute is populated later
# when we have retrieved all Steam apps and can find the
# corresponding app using its app ID
steam_app.required_tool_app = required_tool_app
return steam_app
def __apps_from_file__(path):
"""Read all the apps the user has from sharedconfig.vdf. Returns a dict. Intermidiate function."""
with open(path, encoding='UTF-8') as file:
file_string = file.read()
parsed = vdf.loads(file_string)
return parsed["UserRoamingConfigStore"]["Software"]["Valve"][
"Steam"]["Apps"]
def main():
p = argparse.ArgumentParser(prog='vdf2json')
p.add_argument('infile', nargs='?', type=argparse.FileType('r'),
default=sys.stdin, help="VDF")
p.add_argument('outfile', nargs='?', type=argparse.FileType('w'),
default=sys.stdout, help="JSON (utf8)")
p.add_argument('-p', '--pretty', help='pretty json output', action='store_true')
p.add_argument('-ei', default='utf-8', type=str, metavar='encoding',
help='input encoding E.g.: utf8, utf-16le, etc')
p.add_argument('-eo', default='utf-8', type=str, metavar='encoding',
help='output encoding E.g.: utf8, utf-16le, etc')
args = p.parse_args()
# unicode pain
if args.infile is not sys.stdin:
args.infile.close()
args.infile = codecs.open(args.infile.name, 'r', encoding=args.ei)
else:
args.infile = codecs.getreader(args.ei)(sys.stdin)
if args.outfile is not sys.stdout:
args.outfile.close()
args.outfile = codecs.open(args.outfile.name, 'w', encoding=args.eo)
else:
args.outfile = codecs.getwriter(args.eo)(sys.stdout)
data = vdf.loads(args.infile.read(), mapper=OrderedDict)
json.dump(data, args.outfile, indent=4 if args.pretty else 0, ensure_ascii=False)
def func(
name, appid, compat_tool_name, is_default_proton=True,
library_dir=None, required_tool_app=None):
steam_app = steam_app_factory(
name=name, appid=appid, library_dir=library_dir,
required_tool_app=required_tool_app
)
shutil.rmtree(str(Path(steam_app.prefix_path).parent))
steam_app.prefix_path = None
install_path = Path(steam_app.install_path)
(install_path / "proton").touch()
(install_path / "dist" / "bin").mkdir(parents=True)
(install_path / "dist" / "bin" / "wine").touch()
(install_path / "dist" / "bin" / "wineserver").touch()
# Update config
if is_default_proton:
steam_config = vdf.loads(steam_config_path.read_text())
steam_config["InstallConfigStore"]["Software"]["Valve"]["Steam"][
"CompatToolMapping"]["0"] = {
"name": compat_tool_name,
"config": "",
"Priority": "250"
}
steam_config_path.write_text(vdf.dumps(steam_config))
# Add the Proton installation to the appinfo.vdf, which contains
# a manifest of all official Proton installations
appinfo_factory(
proton_app=steam_app, compat_tool_name=compat_tool_name
)
return steam_app
def get_steam_user_info():
with open(os.path.join(_get_steam_install_location(), r'config\loginusers.vdf'), 'r') as logins_file:
logins = vdf.loads(logins_file.read())
# Return the first user's ID
for user in logins['users']:
return user, logins['users'][user]['AccountName']
def get_change_number(self, appids=None):
if not appids:
appids = []
else:
appids = [appids]
packageids = []
resp = self.steam.send_job_and_wait(
MsgProto(EMsg.ClientPICSProductInfoRequest), {
'apps': map(lambda x: {'appid': x}, appids),
'packages': map(lambda x: {'packageid': x}, packageids),
},
timeout=10)
if not resp:
return {}
resp = proto_to_dict(resp)
for app in resp.get('apps', []):
app['appinfo'] = vdf.loads(
app.pop('buffer').rstrip('\x00'))['appinfo']
app['sha'] = hexlify(app['sha'])
for pkg in resp.get('packages', []):
pkg['appinfo'] = vdf.binary_loads(pkg.pop('buffer')[4:])[str(
pkg['packageid'])]
pkg['sha'] = hexlify(pkg['sha'])
return resp['apps'][0]['change_number']
def find_current_steamid3(steam_path):
"""
Find the SteamID3 of the currently logged in Steam user
"""
def to_steamid3(steamid64):
"""Convert a SteamID64 into the SteamID3 format"""
return int(steamid64) & 0xffffffff
loginusers_path = steam_path / "config" / "loginusers.vdf"
try:
content = loginusers_path.read_text()
vdf_data = vdf.loads(content)
except IOError:
return None
user_datas = [(user_id, lower_dict(user_data))
for user_id, user_data in vdf_data["users"].items()]
users = [{
"steamid3": to_steamid3(user_id),
"account_name": user_data["accountname"],
"timestamp": user_data.get("timestamp", 0)
} for user_id, user_data in user_datas]
# Return the user with the highest timestamp, as that's likely to be the
# currently logged-in user
if users:
user = max(users, key=lambda u: u["timestamp"])
logger.info("Currently logged-in Steam user: %s", user["account_name"])
return user["steamid3"]
return None
def test_keyvalue_open_quoted(self):
INPUT = (
'"key1" "a\n'
'b\n'
'c"\n'
'key2 "a\n'
'b\n'
'c"\n'
)
EXPECTED = {
'key1': 'a\nb\nc',
'key2': 'a\nb\nc',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def get_product_info(self, apps=[], packages=[], timeout=15):
"""Get product info for apps and packages
:param apps: items in the list should be either just ``app_id``, or ``(app_id, access_token)``
:type apps: :class:`list`
:param packages: items in the list should be either just ``package_id``, or ``(package_id, access_token)``
:type packages: :class:`list`
:return: dict with ``apps`` and ``packages`` containing their info, see example below
:rtype: :class:`dict`, :class:`None`
.. code:: python
{'apps': {570: {...}, ...},
'packages': {123: {...}, ...}
}
"""
if not apps and not packages: return
message = MsgProto(EMsg.ClientPICSProductInfoRequest)
for app in apps:
app_info = message.body.apps.add()
app_info.only_public = False
if isinstance(app, tuple):
app_info.appid, app_info.access_token = app
else:
app_info.appid = app
for package in packages:
package_info = message.body.packages.add()
if isinstance(package, tuple):
package_info.appid, package_info.access_token = package
else:
package_info.packageid = package
message.body.meta_data_only = False
job_id = self.send_job(message)
data = dict(apps={}, packages={})
while True:
chunk = self.wait_event(job_id, timeout=timeout)
if chunk is None: return
chunk = chunk[0].body
for app in chunk.apps:
data['apps'][app.appid] = vdf.loads(app.buffer[:-1].decode(
'utf-8', 'replace'))['appinfo']
for pkg in chunk.packages:
data['packages'][pkg.packageid] = vdf.binary_loads(
pkg.buffer[4:])[str(pkg.packageid)]
if not chunk.response_pending:
break
return data
def test_routines_mapper_passing(self, mock_parse):
vdf.load(sys.stdin, mapper=dict)
mock_parse.assert_called_with(sys.stdin, mapper=dict)
vdf.loads("", mapper=dict)
(fp,), kw = mock_parse.call_args
self.assertIsInstance(fp, StringIO)
self.assertIs(kw['mapper'], dict)
class CustomDict(dict):
pass
vdf.load(sys.stdin, mapper=CustomDict)
mock_parse.assert_called_with(sys.stdin, mapper=CustomDict)
vdf.loads("", mapper=CustomDict)
(fp,), kw = mock_parse.call_args
self.assertIsInstance(fp, StringIO)
self.assertIs(kw['mapper'], CustomDict)
def test_merge_multiple_keys_off(self):
INPUT = '''
a
{
a 1
b 2
}
a
{
a 3
c 4
}
'''
EXPECTED = {'a': {'a': '3', 'c': '4'}}
self.assertEqual(vdf.loads(INPUT, merge_duplicate_keys=False), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False, merge_duplicate_keys=False), EXPECTED)
def test_merge_multiple_keys_off(self):
INPUT = '''
a
{
a 1
b 2
}
a
{
a 3
c 4
}
'''
EXPECTED = {'a': {'a': '3', 'c': '4'}}
self.assertEqual(vdf.loads(INPUT, merge_duplicate_keys=False), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False, merge_duplicate_keys=False), EXPECTED)
def test_routines_mapper_passing(self, mock_parse):
vdf.load(sys.stdin, mapper=dict)
mock_parse.assert_called_with(sys.stdin, mapper=dict)
vdf.loads("", mapper=dict)
(fp, ), kw = mock_parse.call_args
self.assertIsInstance(fp, StringIO)
self.assertIs(kw['mapper'], dict)
class CustomDict(dict):
pass
vdf.load(sys.stdin, mapper=CustomDict)
mock_parse.assert_called_with(sys.stdin, mapper=CustomDict)
vdf.loads("", mapper=CustomDict)
(fp, ), kw = mock_parse.call_args
self.assertIsInstance(fp, StringIO)
self.assertIs(kw['mapper'], CustomDict)
def get_product_info(self, apps=[], packages=[], timeout=15):
"""Get product info for apps and packages
:param apps: items in the list should be either just ``app_id``, or ``(app_id, access_token)``
:type apps: :class:`list`
:param packages: items in the list should be either just ``package_id``, or ``(package_id, access_token)``
:type packages: :class:`list`
:return: dict with ``apps`` and ``packages`` containing their info, see example below
:rtype: :class:`dict`, :class:`None`
.. code:: python
{'apps': {570: {...}, ...},
'packages': {123: {...}, ...}
}
"""
if not apps and not packages: return
message = MsgProto(EMsg.ClientPICSProductInfoRequest)
for app in apps:
app_info = message.body.apps.add()
app_info.only_public = False
if isinstance(app, tuple):
app_info.appid, app_info.access_token = app
else:
app_info.appid = app
for package in packages:
package_info = message.body.packages.add()
if isinstance(package, tuple):
package_info.appid, package_info.access_token = package
else:
package_info.packageid = package
message.body.meta_data_only = False
job_id = self.send_job(message)
data = dict(apps={}, packages={})
while True:
chunk = self.wait_event(job_id, timeout=timeout)
if chunk is None: return
chunk = chunk[0].body
for app in chunk.apps:
data['apps'][app.appid] = vdf.loads(app.buffer[:-1].decode('utf-8', 'replace'))['appinfo']
for pkg in chunk.packages:
data['packages'][pkg.packageid] = vdf.binary_loads(pkg.buffer[4:])[str(pkg.packageid)]
if not chunk.response_pending:
break
return data
def test_keyvalue_pairs(self):
INPUT = '''
"key1" "value1"
key2 "value2"
KEY3 "value3"
"key4" value4
"key5" VALUE5
'''
EXPECTED = {
'key1': 'value1',
'key2': 'value2',
'KEY3': 'value3',
'key4': 'value4',
'key5': 'VALUE5',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def test_keyvalue_pairs(self):
INPUT = '''
"key1" "value1"
key2 "value2"
KEY3 "value3"
"key4" value4
"key5" VALUE5
'''
EXPECTED = {
'key1': 'value1',
'key2': 'value2',
'KEY3': 'value3',
'key4': 'value4',
'key5': 'VALUE5',
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def set_last_user(self, username):
reg_file = os.path.expanduser('~/.steam/registry.vdf')
reg_data = vdf.loads(read_file(reg_file))
steam_config = subkey_lookup(reg_data,
r'Registry\HKCU\Software\Valve\Steam')
steam_config['AutoLoginUser'] = username
steam_config['RememberPassword'] = '******'
reg_data = vdf.dumps(reg_data, pretty=True)
with open(reg_file, 'wt') as f:
f.write(reg_data)
def findGamePath() -> Union[Path, None]:
# Try to find the game path through registry entries and library files
from w3modmanager.core.model import verifyGamePath
import winreg
import vdf
try:
# try to read Witcher 3 GOG installation path directly
# see https://www.gog.com/forum/general/querying_gog_install_path
key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE,
r'SOFTWARE',
access=(winreg.KEY_READ
| winreg.KEY_WOW64_64KEY))
subkey = winreg.OpenKeyEx(key, r'WOW6432Node\GOG.com\Games\1207664663')
game = winreg.QueryValueEx(subkey, 'exe')
game = Path(str(game[0]))
if verifyGamePath(game):
return game
except Exception: # noqa
# probably not found
pass
try:
# try to read Steam installation path
# see https://stackoverflow.com/questions/34090258/find-steam-games-folder
key = winreg.OpenKeyEx(winreg.HKEY_LOCAL_MACHINE,
r'SOFTWARE',
access=(winreg.KEY_READ
| winreg.KEY_WOW64_64KEY))
subkey = winreg.OpenKeyEx(key, r'WOW6432Node\Valve\Steam')
steam = winreg.QueryValueEx(subkey, 'installPath')
steam = Path(str(steam[0]))
libs = steam.joinpath('steamapps/libraryfolders.vdf')
if steam.exists() and libs.is_file():
# found Steam installation, now read library folders manifest
# and iterate libraries
libdict = vdf.loads(util.readText(libs), mapper=vdf.VDFDict)
libvals = libdict['LibraryFolders']
for key in libvals:
checkpath = Path(libvals[key])
if checkpath.is_dir() and checkpath.joinpath(
'steamapps').is_dir():
# Steam library path found, now check for Witcher 3 installation
steamapps = checkpath.joinpath('steamapps')
game = steamapps.joinpath(
'common/The Witcher 3/bin/x64/witcher3.exe')
if verifyGamePath(game):
return game
else:
pass
except Exception: # noqa
# probably not found
pass
return None
def test_escape_before_last_unescaped(self):
INPUT = r'''
"aaa\\" "1"
"1" "bbb\\"
'''
EXPECTED = {
"aaa\\\\": "1",
"1": "bbb\\\\",
}
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def __call__(self, handle):
# language = pycountry.languages.get(alpha_2=self.language)
# file_base = copy.deepcopy(vdfTranslationBase)
file_base = vdf.loads(vdf.dumps(vdfTranslationBase)) # copy structure
# file_base['lang']['Language'] = language.name
file_base['lang']['Language'] = self.language.capitalize()
for unit in self.units:
file_base['lang']['Tokens'][unit.key] = unit.text
file_content = vdf.dump(file_base, handle, pretty=True)
def test_comments_and_blank_lines(self):
INPUT = '''
// this is comment
"key1" "value1" // another comment
key2 "value2" // further comments
"key3" value3 // useless comment
key4 // comments comments comments
{ // is this a comment?
k v // comment
} // you only comment once
// comment out of nowhere
"key5" // pretty much anything here
{ // is this a comment?
K V //comment
}
'''
EXPECTED = {
'key1': 'value1',
'key2': 'value2',
'key3': 'value3',
'key4': {
'k': 'v'
},
'key5': {
'K': 'V'
},
}
self.assertEqual(vdf.loads(INPUT), EXPECTED)
self.assertEqual(vdf.loads(INPUT, escaped=False), EXPECTED)
def webapi_request(path, method="GET", caller=None, params={}):
"""
Low level function for calling Steam's WebAPI
"""
if method not in ("GET", "POST"):
raise NotImplemented("HTTP method: %s" % repr(self.method))
onetime = {}
for param in DEFAULT_PARAMS:
params[param] = onetime[param] = params.get(param, DEFAULT_PARAMS[param])
path = "%s://api.steampowered.com/%s" % ("https" if params.get("https", True) else "http", path)
del params["raw"]
del params["https"]
del params["http_timeout"]
if onetime["format"] not in ("json", "vdf", "xml"):
raise ValueError("Expected format to be json,vdf or xml; got %s" % onetime["format"])
# move params to data, if data is not specified for POST
# simplifies code calling this method
kwargs = {"params": params} if method == "GET" else {"data": params}
f = getattr(requests, method.lower())
resp = f(path, stream=True, timeout=onetime["http_timeout"], **kwargs)
if caller is not None:
caller.last_response = resp
if not resp.ok:
raise requests.exceptions.HTTPError("%s %s" % (resp.status_code, resp.reason))
if onetime["raw"]:
return resp.content
if onetime["format"] == "json":
return resp.json()
elif onetime["format"] == "xml":
import lxml.etree
return lxml.etree.parse(resp.raw)
elif onetime["format"] == "vdf":
import vdf
return vdf.loads(resp.text)
def get_product_info(self, appids=[], packageids=[]):
resp = self.steam.send_job_and_wait(MsgProto(EMsg.ClientPICSProductInfoRequest),
{
'apps': map(lambda x: {'appid': x}, appids),
'packages': map(lambda x: {'packageid': x}, packageids),
},
timeout=10
)
if not resp: return {}
resp = proto_to_dict(resp)
for app in resp.get('apps', []):
app['appinfo'] = vdf.loads(app.pop('buffer')[:-1].decode('utf-8', 'replace'))['appinfo']
app['sha'] = hexlify(app['sha']).decode('utf-8')
for pkg in resp.get('packages', []):
pkg['appinfo'] = vdf.binary_loads(pkg.pop('buffer')[4:])[str(pkg['packageid'])]
pkg['sha'] = hexlify(pkg['sha']).decode('utf-8')
return resp
def requestGetSchema():
URL = "https://api.steampowered.com/IEconItems_570/GetSchemaURL/v1?key=" + SteamAPIKey
print(URL)
response = requests.get(URL)
response.connection.close()
response = response.json()
print(response)
global dota2schema
URL = response['result']['items_game_url']
with open ("items_game_url.txt", "r") as text_file:
data=text_file.read()
print(data)
print(URL)
if (data != URL):
response = requests.get(URL)
response.connection.close()
response = response.text
dota2schema = vdf.loads(response)
with open('dota2schema.txt', 'w') as outfile:
json.dump(dota2schema, outfile)
with open("items_game_url.txt", "w") as text_file:
text_file.write(URL)
else:
with open ("dota2schema.txt", "r") as text_file:
data=text_file.read()
dota2schema = json.loads(data)
GetHeroes = [{"name": "Blank", "id": 0}] # Have the very first entry be blank.
GetHeroes.extend(json.loads(urllib2.urlopen("http://api.steampowered.com/IEconDOTA2_570/GetHeroes/v0001/?key=%(key)s" % {"key": _API_KEY}).read())["result"]["heroes"])
parseHeroes = []
for x in xrange(0, len(GetHeroes)):
found = False
for item in GetHeroes:
if item["id"] == x:
parseHeroes.append(item)
found = True
if found == False:
parseHeroes.append({"name": "FAILBOAT", "id": x})
mod_texture = vdf.loads(open("input/mod_textures.txt", "r").read().replace("TextureData", "\"TextureData\"")) # Case-specific hack due to vdf.py being more strongly syntax'd than Valve's own VDFs.
raw_spritesheet = Image.open("input/minimap_hero_sheet.png")
raw_sprite_data = {}
for entry in mod_texture["sprites/640_hud"]["TextureData"].iteritems():
if entry[0][:16] == "minimap_heroicon":
raw_sprite_data[entry[0]] = entry[1]
raw_sprites = {}
for data in raw_sprite_data.iteritems():
raw_sprites[data[0][17:]] = raw_spritesheet.crop((data[1]["x"], data[1]["y"], data[1]["x"] + data[1]["width"], data[1]["y"] + data[1]["height"]))
output_image_width = ROW_LIMIT * SPRITE_DESIRED_WIDTH
output_image_height = ((len(parseHeroes) / ROW_LIMIT) * SPRITE_DESIRED_HEIGHT)
def test_routine_loads(self, mock_parse):
vdf.loads("")
(fp,), _ = mock_parse.call_args
self.assertIsInstance(fp, StringIO)
def main():
with open("npc_heroes.txt", "r") as f:
hero_data = vdf.loads(f.read())["DOTAHeroes"]
heroes = json.loads(urllib2.urlopen("http://api.steampowered.com/IEconDOTA2_570/GetHeroes/v1/?key={}&language=en_us&itemizedonly".format(config["webapi_key"])).read())["result"]["heroes"]
w = wiki.Wiki(config["api"])
report_data = {}
for hero in heroes:
print("Parsing {}".format(hero["localized_name"]))
report_data[hero["localized_name"]] = [] # Set up array for hero attribute tuples
# Data from Wiki
text = page.Page(w, title=hero["localized_name"]).getWikiText()
infobox_params = infobox_re.search(text).group(1)
params = {k.lower(): v for k, v in param_re.findall(infobox_params)}
# Data from npc_heroes.txt
vdf_base_data = hero_data["npc_dota_hero_base"]
vdf_hero_data = hero_data[hero["name"]]
# Do comparisons
def compare(tuple1, tuple2):
special_cases = ["armor", "missile speed", "damage max", "damage min"]
k1, v1 = tuple1
k2, v2 = tuple2
if k1 not in special_cases:
return (float(v1) == float(v2)), tuple1, tuple2
else:
if k1 == "armor":
# Wiki has armor specified with Agi calculated in.
agi_boost = 0.14 * int(vdf_hero_data["AttributeBaseAgility"])
return (float(v1) == (float(v2) + agi_boost)), tuple1, (k2, float(v2) + agi_boost)
elif k1 == "missile speed":
if v1 == -1 and vdf_hero_data.get("ProjectileSpeed") in [0, None, ""]:
# If we don't have a value listed on Wiki and it is set to 0 or not set at all, treat as a match.
return True, tuple1, tuple2
else:
return (float(v1) == float(v2)), tuple1, tuple2
elif k1 == "damage min" or k1 == "damage max":
# Wiki has damage with primary attr damage calcualted in.
if vdf_hero_data["AttributePrimary"] == "DOTA_ATTRIBUTE_AGILITY":
primary_attr_dmg = int(vdf_hero_data["AttributeBaseAgility"])
elif vdf_hero_data["AttributePrimary"] == "DOTA_ATTRIBUTE_STRENGTH":
primary_attr_dmg = int(vdf_hero_data["AttributeBaseStrength"])
else:
primary_attr_dmg = int(vdf_hero_data["AttributeBaseIntelligence"])
return (float(v1) == (float(v2) + primary_attr_dmg)), tuple1, (k2, float(v2) + primary_attr_dmg)
for a, b in [
# ("attack backswing", ""), # Not sure where this data is from.
# ("cast backswing", ""),
# ("cast point", ""),
("agility", "AttributeBaseAgility"),
("agility growth", "AttributeAgilityGain"),
("armor", "ArmorPhysical"),
("attack point", "AttackAnimationPoint"),
("attack range", "AttackRange"),
("bat", "AttackRate"),
("damage max", "AttackDamageMax"),
("damage min", "AttackDamageMin"),
("intelligence", "AttributeBaseIntelligence"),
("intelligence growth", "AttributeIntelligenceGain"),
("movement speed", "MovementSpeed"),
("missile speed", "ProjectileSpeed"),
("sight range day", "VisionDaytimeRange"),
("sight range night", "VisionNighttimeRange"),
("strength", "AttributeBaseStrength"),
("strength growth", "AttributeStrengthGain"),
("turn rate", "MovementTurnRate"),
]:
val1 = params[a] if a in params else -1 # If we don't use this attr in the Wiki replace with -1
val2 = vdf_hero_data.get(b, vdf_base_data.get(b)) # Fall back to npc_dota_hero_base if attr not defined in hero spec.
report_data[hero["localized_name"]].append(compare((a, val1), (b, val2)))
report_to_wicky(w, report_data)
def test_empty(self):
self.assertEqual(vdf.loads(""), {})
def webapi_request(url, method="GET", caller=None, session=None, params=None):
"""Low level function for calling Steam's WebAPI
.. versionchanged:: 0.8.3
:param url: request url (e.g. ``https://api.steampowered.com/A/B/v001/``)
:type url: :class:`str`
:param method: HTTP method (GET or POST)
:type method: :class:`str`
:param caller: caller reference, caller.last_response is set to the last response
:param params: dict of WebAPI and endpoint specific params
:type params: :class:`dict`
:param session: an instance requests session, or one is created per call
:type session: :class:`requests.Session`
:return: response based on paramers
:rtype: :class:`dict`, :class:`lxml.etree.Element`, :class:`str`
"""
if method not in ("GET", "POST"):
raise NotImplemented("HTTP method: %s" % repr(self.method))
if params is None:
params = {}
onetime = {}
for param in DEFAULT_PARAMS:
params[param] = onetime[param] = params.get(param, DEFAULT_PARAMS[param])
for param in ("raw", "apihost", "https", "http_timeout"):
del params[param]
if onetime["format"] not in ("json", "vdf", "xml"):
raise ValueError("Expected format to be json,vdf or xml; got %s" % onetime["format"])
for k, v in list(params.items()): # serialize some types
if isinstance(v, bool):
params[k] = 1 if v else 0
elif isinstance(v, dict):
params[k] = _json.dumps(v)
elif isinstance(v, list):
del params[k]
for i, lvalue in enumerate(v):
params["%s[%d]" % (k, i)] = lvalue
kwargs = {"params": params} if method == "GET" else {"data": params} # params to data for POST
if session is None:
session = _make_session()
f = getattr(session, method.lower())
resp = f(url, stream=False, timeout=onetime["http_timeout"], **kwargs)
# we keep a reference of the last response instance on the caller
if caller is not None:
caller.last_response = resp
# 4XX and 5XX will cause this to raise
resp.raise_for_status()
if onetime["raw"]:
return resp.text
elif onetime["format"] == "json":
return resp.json()
elif onetime["format"] == "xml":
from lxml import etree as _etree
return _etree.fromstring(resp.content)
elif onetime["format"] == "vdf":
import vdf as _vdf
return _vdf.loads(resp.text)
def copy(self, obj=None): """Create a copy of a VDFDict object.""" if obj is None: obj = self.get_data() return vdf.loads(vdf.dumps(obj, pretty=True), mapper=vdf.VDFDict)
def test_routine_loads(self, mock_parse):
vdf.loads("")
mock_parse.assert_called_with("")
