def download_via_url(args, pubfile):
sess = make_requests_session()
fstream = sess.get(pubfile['file_url'], stream=True)
total_size = int(pubfile['file_size'])
relpath = sanitizerelpath(pubfile['filename'])
if args.no_directories:
relpath = os.path.basename(relpath)
relpath = os.path.join(args.output, relpath)
filepath = os.path.abspath(relpath)
ensure_dir(filepath)
with open(filepath, 'wb') as fp:
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(total=total_size, unit='B', unit_scale=True)
gevent.spawn(pbar.gevent_refresh_loop)
else:
pbar = fake_tqdm()
LOG.info('Downloading to {} ({})'.format(
relpath,
fmt_size(total_size),
))
for chunk in iter(lambda: fstream.raw.read(1024**2), b''):
fp.write(chunk)
pbar.update(len(chunk))
pbar.close()
def download_via_steampipe(args, pubfile):
from steamctl.clients import CachingSteamClient
s = CachingSteamClient()
if args.cell_id is not None:
s.cell_id = args.cell_id
cdn = s.get_cdnclient()
key = pubfile['consumer_appid'], pubfile['consumer_appid'], pubfile[
'hcontent_file']
# only login if we dont have depot decryption key
if int(pubfile['consumer_appid']) not in cdn.depot_keys:
result = s.login_from_args(args)
if result == EResult.OK:
LOG.info("Login to Steam successful")
else:
LOG.error("Failed to login: %r" % result)
return 1 # error
try:
manifest = cdn.get_manifest(*key)
except SteamError as exp:
if exp.eresult == EResult.AccessDenied:
LOG.error("This account doesn't have access to the app depot")
else:
LOG.error(str(exp))
return 1 # error
else:
manifest.name = pubfile['title']
LOG.debug("Got manifest: %r", manifest)
LOG.info("File manifest acquired")
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(total=manifest.size_original, unit='B', unit_scale=True)
gevent.spawn(pbar.gevent_refresh_loop)
else:
pbar = fake_tqdm()
tasks = GPool(4)
for mfile in manifest:
if not mfile.is_file:
continue
tasks.spawn(mfile.download_to,
args.output,
no_make_dirs=args.no_directories,
pbar=pbar)
# wait on all downloads to finish
tasks.join()
# clean and exit
pbar.close()
cdn.save_cache()
s.disconnect()
LOG.info("Download complete.")
def cmd_cloud_download(args):
with init_client(args) as s:
files, total_files, total_size = get_cloud_files(s, args.app_id)
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(desc='Data ',
mininterval=0.5,
maxinterval=1,
miniters=1024**3 * 10,
total=total_size,
unit='B',
unit_scale=True)
pbar2 = tqdm(desc='Files',
mininterval=0.5,
maxinterval=1,
miniters=10,
total=total_files,
position=1,
unit=' file',
unit_scale=False)
gevent.spawn(pbar.gevent_refresh_loop)
gevent.spawn(pbar2.gevent_refresh_loop)
else:
pbar = fake_tqdm()
pbar2 = fake_tqdm()
tasks = GPool(6)
sess = make_requests_session()
for entry in files:
tasks.spawn(download_file, args, sess, entry, pbar, pbar2)
tasks.join()
pbar.refresh()
pbar2.refresh()
pbar.close()
def download_via_url(args, url, filename):
sess = make_requests_session()
fstream = sess.get(url, stream=True)
total_size = int(fstream.headers.get('Content-Length', 0))
relpath = sanitizerelpath(filename)
if args.no_directories:
relpath = os.path.basename(relpath)
relpath = os.path.join(args.output, relpath)
filepath = os.path.abspath(relpath)
ensure_dir(filepath)
with open(filepath, 'wb') as fp:
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(total=total_size,
mininterval=0.5,
maxinterval=1,
miniters=1024**3 * 10,
unit='B',
unit_scale=True)
gevent.spawn(pbar.gevent_refresh_loop)
else:
pbar = fake_tqdm()
# LOG.info('Downloading to {} ({})'.format(
# relpath,
# fmt_size(total_size) if total_size else 'Unknown size',
# ))
for chunk in iter(lambda: fstream.raw.read(8388608), b''):
fp.write(chunk)
pbar.update(len(chunk))
pbar.close()
def cmd_depot_download(args):
pbar = fake_tqdm()
pbar2 = fake_tqdm()
try:
with init_clients(args) as (_, _, manifests):
fileindex = ManifestFileIndex(manifests)
# pre-index vpk file to speed up lookups
if args.vpk:
fileindex.index('*.vpk')
# calculate total size
total_files = 0
total_size = 0
LOG.info("Locating and counting files...")
for manifest in manifests:
for depotfile in manifest:
if not depotfile.is_file:
continue
filepath = depotfile.filename_raw
# list files inside vpk
if args.vpk and filepath.endswith('.vpk'):
# fast skip VPKs that can't possibly match
if args.name and ':' in args.name:
pre = args.name.split(':', 1)[0]
if not fnmatch(filepath, pre):
continue
if args.regex and ':' in args.regex:
pre = args.regex.split(':', 1)[0]
if not re_search(pre + '$', filepath):
continue
# scan VPKs, but skip data only ones
if filepath.endswith('_dir.vpk') or not re.search(
"_\d+\.vpk$", filepath):
LOG.debug("Scanning VPK file: %s", filepath)
try:
fvpk = fileindex.get_vpk(filepath)
except ValueError as exp:
LOG.error("VPK read error: %s", str(exp))
else:
for vpkfile_path, (
_, _, _, _, _,
size) in fvpk.c_iter_index():
complete_path = "{}:{}".format(
filepath, vpkfile_path)
if args.name and not fnmatch(
complete_path, args.name):
continue
if args.regex and not re_search(
args.regex, complete_path):
continue
total_files += 1
total_size += size
# account for depot files
if args.name and not fnmatch(filepath, args.name):
continue
if args.regex and not re_search(args.regex, filepath):
continue
total_files += 1
total_size += depotfile.size
if not total_files:
raise SteamError("No files found to download")
# enable progress bar
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(desc='Data ',
mininterval=0.5,
maxinterval=1,
total=total_size,
unit='B',
unit_scale=True)
pbar2 = tqdm(desc='Files',
mininterval=0.5,
maxinterval=1,
total=total_files,
position=1,
unit=' file',
unit_scale=False)
gevent.spawn(pbar.gevent_refresh_loop)
gevent.spawn(pbar2.gevent_refresh_loop)
# download files
tasks = GPool(6)
for manifest in manifests:
if pbar2.n == total_files:
break
LOG.info("Processing manifest (%s) '%s' ..." %
(manifest.gid, manifest.name or "<Unknown>"))
for depotfile in manifest:
if pbar2.n == total_files:
break
if not depotfile.is_file:
continue
filepath = depotfile.filename_raw
if args.vpk and filepath.endswith('.vpk'):
# fast skip VPKs that can't possibly match
if args.name and ':' in args.name:
pre = args.name.split(':', 1)[0]
if not fnmatch(filepath, pre):
continue
if args.regex and ':' in args.regex:
pre = args.regex.split(':', 1)[0]
if not re_search(pre + '$', filepath):
continue
# scan VPKs, but skip data only ones
if filepath.endswith('_dir.vpk') or not re.search(
"_\d+\.vpk$", filepath):
LOG.debug("Scanning VPK file: %s", filepath)
try:
fvpk = fileindex.get_vpk(filepath)
except ValueError as exp:
LOG.error("VPK read error: %s", str(exp))
else:
for vpkfile_path, metadata in fvpk.c_iter_index(
):
complete_path = "{}:{}".format(
filepath, vpkfile_path)
if args.name and not fnmatch(
complete_path, args.name):
continue
if args.regex and not re_search(
args.regex, complete_path):
continue
tasks.spawn(
vpkfile_download_to,
depotfile.filename,
fvpk.get_vpkfile_instance(
vpkfile_path,
fvpk._make_meta_dict(metadata)),
args.output,
no_make_dirs=args.no_directories,
pbar=pbar,
)
pbar2.update(1)
# break out of vpk file loop
if pbar2.n == total_files:
break
# break out of depotfile loop
if pbar2.n == total_files:
break
# filepath filtering
if args.name and not fnmatch(filepath, args.name):
continue
if args.regex and not re_search(args.regex, filepath):
continue
tasks.spawn(
depotfile.download_to,
args.output,
no_make_dirs=args.no_directories,
pbar=pbar,
verify=(not args.skip_verify),
)
pbar2.update(1)
# wait on all downloads to finish
tasks.join()
gevent.sleep(0.5)
except KeyboardInterrupt:
pbar.close()
LOG.info("Download canceled")
return 1 # error
except SteamError as exp:
pbar.close()
pbar.write(str(exp))
return 1 # error
else:
pbar.close()
if not args.no_progress:
pbar2.close()
pbar2.write('\n')
LOG.info('Download complete')
def cmd_depot_download(args):
pbar = fake_tqdm()
pbar2 = fake_tqdm()
try:
with init_clients(args) as (s, cdn, manifests):
# calculate total size
if not args.no_progress or args.name or args.regex:
total_files = 0
total_size = 0
for manifest in manifests:
for depotfile in manifest:
if not depotfile.is_file:
continue
if args.name and not fnmatch(depotfile.filename, args.name):
continue
if args.regex and not re_search(args.regex, depotfile.filename):
continue
total_files += 1
total_size += depotfile.size
else:
total_files = sum(map(lambda x: len(x), manifests))
total_size = sum(map(lambda x: x.size_original, manifests))
# enable progress bar
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(desc='Downloaded', mininterval=0.5, maxinterval=1, total=total_size, unit=' B', unit_scale=True)
pbar2 = tqdm(desc='Files ', mininterval=0.5, maxinterval=1, total=total_files, position=1, unit=' file', unit_scale=False)
gevent.spawn(pbar.gevent_refresh_loop)
gevent.spawn(pbar2.gevent_refresh_loop)
# download files
tasks = GPool(4)
for manifest in manifests:
LOG.info("Processing (%s) '%s' ..." % (manifest.gid, manifest.name))
for depotfile in manifest:
if not depotfile.is_file:
continue
# filepath filtering
if args.name and not fnmatch(depotfile.filename, args.name):
continue
if args.regex and not re_search(args.regex, depotfile.filename):
continue
tasks.spawn(depotfile.download_to, args.output,
no_make_dirs=args.no_directories,
pbar=pbar)
pbar2.update(1)
# wait on all downloads to finish
tasks.join()
gevent.sleep(0.5)
except KeyboardInterrupt:
pbar.close()
LOG.info("Download canceled")
return 1 # error
except SteamError as exp:
pbar.close()
pbar.write(str(exp))
return 1 # error
else:
pbar.close()
if not args.no_progress:
pbar2.close()
pbar2.write('\n')
LOG.info('Download complete')