def download_via_steampipe(args, pubfile):
from steamctl.clients import CachingSteamClient
s = CachingSteamClient()
if args.cell_id is not None:
s.cell_id = args.cell_id
cdn = s.get_cdnclient()
key = pubfile['consumer_appid'], pubfile['consumer_appid'], pubfile[
'hcontent_file']
# only login if we dont have depot decryption key
if int(pubfile['consumer_appid']) not in cdn.depot_keys:
result = s.login_from_args(args)
if result == EResult.OK:
LOG.info("Login to Steam successful")
else:
LOG.error("Failed to login: %r" % result)
return 1 # error
try:
manifest = cdn.get_manifest(*key)
except SteamError as exp:
if exp.eresult == EResult.AccessDenied:
LOG.error("This account doesn't have access to the app depot")
else:
LOG.error(str(exp))
return 1 # error
else:
manifest.name = pubfile['title']
LOG.debug("Got manifest: %r", manifest)
LOG.info("File manifest acquired")
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(total=manifest.size_original, unit='B', unit_scale=True)
gevent.spawn(pbar.gevent_refresh_loop)
else:
pbar = fake_tqdm()
tasks = GPool(4)
for mfile in manifest:
if not mfile.is_file:
continue
tasks.spawn(mfile.download_to,
args.output,
no_make_dirs=args.no_directories,
pbar=pbar)
# wait on all downloads to finish
tasks.join()
# clean and exit
pbar.close()
cdn.save_cache()
s.disconnect()
LOG.info("Download complete.")
def set_actor_info(events):
actor = events[0]['actor']
github_bot_id = random.randrange(len(githubs))
github = githubs[github_bot_id]
try:
actor_obj = github.get_user(actor)
except UnknownObjectException:
for event in events:
event['actor-disabled'] = True
event['actor-following'] = []
return events
following_count = actor_obj.following
following_pages = int(ceil(following_count / 30.0))
get_following_pagination = actor_obj.get_following()
pool = GPool()
following = pool.map(
partial(get_following, actor, get_following_pagination),
range(following_pages))
for event in events:
event['actor-following'] = list(flatten(following))
event['actor-hireable'] = actor_obj.hireable
event['actor-location'] = actor_obj.location
event['actor-type'] = actor_obj.type
return events
def __init__(self, client):
"""CDNClient allows loading and reading of manifests for Steam apps are used
to list and download content
:param client: logged in SteamClient instance
:type client: :class:`.SteamClient`
"""
self.gpool = GPool(8) #: task pool
self.steam = client #: SteamClient instance
if self.steam:
self.cell_id = self.steam.cell_id
self.web = make_requests_session()
self.depot_keys = {} #: depot decryption keys
self.manifests = {} #: CDNDepotManifest instances
self.app_depots = {} #: app depot info
self.beta_passwords = {} #: beta branch decryption keys
self.licensed_app_ids = set(
) #: app_ids that the SteamClient instance has access to
self.licensed_depot_ids = set(
) #: depot_ids that the SteamClient instance has access to
if not self.servers:
self.fetch_content_servers()
self.load_licenses()
def decorator(f):
pool = GPool(n)
@wraps(f)
def wrapped(*args, **kwargs):
return GTomorrow(pool.spawn(f, *args, **kwargs), timeout)
return wrapped
def reinit(self, cdnapi, options):
self.cdnapi = cdnapi
self.options = options
self.appid = options["appid"]
self.download_directory = Path(
options["download_directory"]).absolute()
self.download_pool = GPool(
16) #ThreadPoolExecutor(16, initializer=thread_init)
def gevent_req(num_req):
from gevent.pool import Pool as GPool
import gevent
pool = GPool(num_req / 2)
glets = []
for x in range(0, num_req):
with gevent.Timeout(10, False):
g = pool.spawn(time_request)
glets.append(g)
pool.join()
return [g.value for g in glets]
def __init__(self, cdnapi, options=None):
if options is None:
options = self.default_options
self.options = options
self.cdnapi = cdnapi
self.appid = options["appid"]
self.download_directory = Path(
options["download_directory"]).absolute()
# everything below here is just default empty state
self.branch_actions = {}
self.download_pool = GPool(
16) #ThreadPoolExecutor(16, initializer=thread_init)
self.last_check = 0
self.check_interval = 5 # seconds
self.change_events = deque()
self.last_manifests = []
self.last_depotinfo = {
} # the entire depot info for this appid, containing all branches.
self.last_branchinfo = defaultdict(self.default_branchinfo)
self.last_branchmanifests = defaultdict(
self.default_none
) # {'noitabeta': [CDNDepotManifest instances], 'never_seen_this_branch': None}
self.current_branchmanifests = defaultdict(
self.default_none) # passing around args between members sucks
# initialize the state for branch tracking
for branch, func in options["branch_actions"].items():
self.branch_actions[branch] = func
#self.last_branchinfo[branch] = {'buildid': '0', 'timeupdated': '0'}
# test code to trigger an update on first check
if False:
self.last_branchinfo['noitabeta'] = {
'buildid': '4246174',
'description': 'Beta branch with experimental modding support',
'timeupdated': '1570816042'
}
def cmd_cloud_download(args):
with init_client(args) as s:
files, total_files, total_size = get_cloud_files(s, args.app_id)
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(desc='Data ',
mininterval=0.5,
maxinterval=1,
miniters=1024**3 * 10,
total=total_size,
unit='B',
unit_scale=True)
pbar2 = tqdm(desc='Files',
mininterval=0.5,
maxinterval=1,
miniters=10,
total=total_files,
position=1,
unit=' file',
unit_scale=False)
gevent.spawn(pbar.gevent_refresh_loop)
gevent.spawn(pbar2.gevent_refresh_loop)
else:
pbar = fake_tqdm()
pbar2 = fake_tqdm()
tasks = GPool(6)
sess = make_requests_session()
for entry in files:
tasks.spawn(download_file, args, sess, entry, pbar, pbar2)
tasks.join()
pbar.refresh()
pbar2.refresh()
pbar.close()
item_q(".rt:first em:last").text()
])
url = "www.bttiantang.com{}".format(item_q("a:first").attr("href"))
if title and date and db_score:
print "<===>{}<===>{}<===>{}<===>{}".format(
title.encode("utf8"), date.encode("utf8"),
db_score.encode("utf8"), url.encode("utf8"))
@_crawl_deco
def _crawl_url(url):
respon = requests.get(url, timeout=3)
if respon.ok:
_parse_html(respon.content)
_crawl_url(url)
if __name__ == '__main__':
start_time = time.time()
crawl_list = ("http://www.bttiantang.com/?PageNo={}".format(i)
for i in range(1, 500))
g_pool = GPool(50)
g_pool.map(run_worker, crawl_list)
end_time = time.time()
print "total cost {} sec.".format(end_time - start_time)
import time
from multiprocessing.pool import Pool as MPool
from gevent.pool import Pool as GPool
def echo(i):
time.sleep(0.001)
return i
p = MPool(10)
run1 = [a for a in p.imap_unordered(echo, range(10))]
run2 = [a for a in p.imap_unordered(echo, range(10))]
run3 = [a for a in p.imap_unordered(echo, range(10))]
run4 = [a for a in p.imap_unordered(echo, range(10))]
print(run1 == run2 == run3 == run4)
p = GPool(10)
run1 = [a for a in p.imap_unordered(echo, range(10))]
run2 = [a for a in p.imap_unordered(echo, range(10))]
run3 = [a for a in p.imap_unordered(echo, range(10))]
run4 = [a for a in p.imap_unordered(echo, range(10))]
print(run1 == run2 == run3 == run4)
def cmd_depot_download(args):
pbar = fake_tqdm()
pbar2 = fake_tqdm()
try:
with init_clients(args) as (_, _, manifests):
fileindex = ManifestFileIndex(manifests)
# pre-index vpk file to speed up lookups
if args.vpk:
fileindex.index('*.vpk')
# calculate total size
total_files = 0
total_size = 0
LOG.info("Locating and counting files...")
for manifest in manifests:
for depotfile in manifest:
if not depotfile.is_file:
continue
filepath = depotfile.filename_raw
# list files inside vpk
if args.vpk and filepath.endswith('.vpk'):
# fast skip VPKs that can't possibly match
if args.name and ':' in args.name:
pre = args.name.split(':', 1)[0]
if not fnmatch(filepath, pre):
continue
if args.regex and ':' in args.regex:
pre = args.regex.split(':', 1)[0]
if not re_search(pre + '$', filepath):
continue
# scan VPKs, but skip data only ones
if filepath.endswith('_dir.vpk') or not re.search(
"_\d+\.vpk$", filepath):
LOG.debug("Scanning VPK file: %s", filepath)
try:
fvpk = fileindex.get_vpk(filepath)
except ValueError as exp:
LOG.error("VPK read error: %s", str(exp))
else:
for vpkfile_path, (
_, _, _, _, _,
size) in fvpk.c_iter_index():
complete_path = "{}:{}".format(
filepath, vpkfile_path)
if args.name and not fnmatch(
complete_path, args.name):
continue
if args.regex and not re_search(
args.regex, complete_path):
continue
total_files += 1
total_size += size
# account for depot files
if args.name and not fnmatch(filepath, args.name):
continue
if args.regex and not re_search(args.regex, filepath):
continue
total_files += 1
total_size += depotfile.size
if not total_files:
raise SteamError("No files found to download")
# enable progress bar
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(desc='Data ',
mininterval=0.5,
maxinterval=1,
total=total_size,
unit='B',
unit_scale=True)
pbar2 = tqdm(desc='Files',
mininterval=0.5,
maxinterval=1,
total=total_files,
position=1,
unit=' file',
unit_scale=False)
gevent.spawn(pbar.gevent_refresh_loop)
gevent.spawn(pbar2.gevent_refresh_loop)
# download files
tasks = GPool(6)
for manifest in manifests:
if pbar2.n == total_files:
break
LOG.info("Processing manifest (%s) '%s' ..." %
(manifest.gid, manifest.name or "<Unknown>"))
for depotfile in manifest:
if pbar2.n == total_files:
break
if not depotfile.is_file:
continue
filepath = depotfile.filename_raw
if args.vpk and filepath.endswith('.vpk'):
# fast skip VPKs that can't possibly match
if args.name and ':' in args.name:
pre = args.name.split(':', 1)[0]
if not fnmatch(filepath, pre):
continue
if args.regex and ':' in args.regex:
pre = args.regex.split(':', 1)[0]
if not re_search(pre + '$', filepath):
continue
# scan VPKs, but skip data only ones
if filepath.endswith('_dir.vpk') or not re.search(
"_\d+\.vpk$", filepath):
LOG.debug("Scanning VPK file: %s", filepath)
try:
fvpk = fileindex.get_vpk(filepath)
except ValueError as exp:
LOG.error("VPK read error: %s", str(exp))
else:
for vpkfile_path, metadata in fvpk.c_iter_index(
):
complete_path = "{}:{}".format(
filepath, vpkfile_path)
if args.name and not fnmatch(
complete_path, args.name):
continue
if args.regex and not re_search(
args.regex, complete_path):
continue
tasks.spawn(
vpkfile_download_to,
depotfile.filename,
fvpk.get_vpkfile_instance(
vpkfile_path,
fvpk._make_meta_dict(metadata)),
args.output,
no_make_dirs=args.no_directories,
pbar=pbar,
)
pbar2.update(1)
# break out of vpk file loop
if pbar2.n == total_files:
break
# break out of depotfile loop
if pbar2.n == total_files:
break
# filepath filtering
if args.name and not fnmatch(filepath, args.name):
continue
if args.regex and not re_search(args.regex, filepath):
continue
tasks.spawn(
depotfile.download_to,
args.output,
no_make_dirs=args.no_directories,
pbar=pbar,
verify=(not args.skip_verify),
)
pbar2.update(1)
# wait on all downloads to finish
tasks.join()
gevent.sleep(0.5)
except KeyboardInterrupt:
pbar.close()
LOG.info("Download canceled")
return 1 # error
except SteamError as exp:
pbar.close()
pbar.write(str(exp))
return 1 # error
else:
pbar.close()
if not args.no_progress:
pbar2.close()
pbar2.write('\n')
LOG.info('Download complete')
def cmd_depot_download(args):
pbar = fake_tqdm()
pbar2 = fake_tqdm()
try:
with init_clients(args) as (s, cdn, manifests):
# calculate total size
if not args.no_progress or args.name or args.regex:
total_files = 0
total_size = 0
for manifest in manifests:
for depotfile in manifest:
if not depotfile.is_file:
continue
if args.name and not fnmatch(depotfile.filename, args.name):
continue
if args.regex and not re_search(args.regex, depotfile.filename):
continue
total_files += 1
total_size += depotfile.size
else:
total_files = sum(map(lambda x: len(x), manifests))
total_size = sum(map(lambda x: x.size_original, manifests))
# enable progress bar
if not args.no_progress and sys.stderr.isatty():
pbar = tqdm(desc='Downloaded', mininterval=0.5, maxinterval=1, total=total_size, unit=' B', unit_scale=True)
pbar2 = tqdm(desc='Files ', mininterval=0.5, maxinterval=1, total=total_files, position=1, unit=' file', unit_scale=False)
gevent.spawn(pbar.gevent_refresh_loop)
gevent.spawn(pbar2.gevent_refresh_loop)
# download files
tasks = GPool(4)
for manifest in manifests:
LOG.info("Processing (%s) '%s' ..." % (manifest.gid, manifest.name))
for depotfile in manifest:
if not depotfile.is_file:
continue
# filepath filtering
if args.name and not fnmatch(depotfile.filename, args.name):
continue
if args.regex and not re_search(args.regex, depotfile.filename):
continue
tasks.spawn(depotfile.download_to, args.output,
no_make_dirs=args.no_directories,
pbar=pbar)
pbar2.update(1)
# wait on all downloads to finish
tasks.join()
gevent.sleep(0.5)
except KeyboardInterrupt:
pbar.close()
LOG.info("Download canceled")
return 1 # error
except SteamError as exp:
pbar.close()
pbar.write(str(exp))
return 1 # error
else:
pbar.close()
if not args.no_progress:
pbar2.close()
pbar2.write('\n')
LOG.info('Download complete')
