def job_spider(cluster, jobs: list = None):
jobs = na_lazy(jobs, cluster.rest_api_job_list)
to_screen("{} jobs to be captured in the cluster {}".format(
len(jobs), cluster.alias))
job_statuses = concurrent_map(
lambda j: cluster.rest_api_job_info(
j['name'], info=None, user=j['username']), jobs)
job_configs = concurrent_map(
lambda j: cluster.rest_api_job_info(
j['name'], info='config', user=j['username']), jobs)
job_logs = concurrent_map(JobStatusParser.all_tasks_logs, job_statuses)
for job, sta, cfg, logs in zip(jobs, job_statuses, job_configs, job_logs):
job['status'] = sta
job['config'] = cfg
job['logs'] = logs
return jobs
def available_resources(self):
"""concurrent version to get available resources"""
aliases = self.aliases
ret = concurrent_map(Cluster.available_resources,
(self.get_client(a) for a in aliases))
return {a: r for a, r in zip(aliases, ret) if r is not None}