def async_obj_map(obj_list, async_fun, async_func_args=[], async_func_kwargs={}, label=None, timing=False, sleep=0.1): if timing: start = time.time() future = {} for key in obj_list: future[key] = async_fun(key, *async_func_args, **async_func_kwargs) output = {} failed = {} with click.progressbar(label=label, length=len(future)) as bar: while future: remain = future future = {} for key, f in remain.items(): if f.done(): try: output[key] = f.result() except Exception as err: failed[key] = err finally: bar.update(1) else: future[key] = f time.sleep(sleep) if timing: end = time.time() dur = end - start dur_str = "Dur: {}".format(util_cli.duration_to_str(dur)) ops = len(obj_list)/dur ops_str = "Objs/sec: {:6.0f}".format(ops) offset = "{val:{width}s}".format(val="", width=(len(label)+1)) click.echo("{} {}, {}".format(offset, dur_str, ops_str)) return output, failed
def util_download_submissions(obj, dest_dir, asn_list, sub_list, usr_list, full_uuid, full_name, timing, overwrite): # Start Timing if timing: start = time.time() # Make Async Calls with obj['connection']: # Fetch Assignments tup = async_obj_fetch([None], obj_name="Assignments", timing=timing, async_list=obj['assignments'].async_list_by_null, async_show=obj['assignments'].async_show, prefilter_list=asn_list) asn_lsts, asn_set, asn_objs, asn_lsts_failed, asn_objs_failed = tup # Fetch Submissions tup = async_obj_fetch(asn_set, obj_name="Submissions", timing=timing, async_list=obj['submissions'].async_list_by_asn, async_show=obj['submissions'].async_show, prefilter_list=sub_list, postfilter_func=postfilter_attr_owner, postfilter_func_args=[usr_list]) sub_lsts, sub_set, sub_objs, sub_lsts_failed, sub_objs_failed = tup # Fetch Files tup = async_obj_fetch(sub_set, obj_name="Files ", timing=timing, async_list=obj['files'].async_list_by_sub, async_show=obj['files'].async_show) fle_lsts, fle_set, fle_objs, fle_lsts_failed, fle_objs_failed = tup # Fetch Users usr_set = set() for sub in sub_objs.values(): usr_set.add(sub["owner"]) usr_objs, usr_objs_failed = async_obj_map(usr_set, obj['users'].async_show, label="Getting Users ", timing=timing) # Build File Lists paths_map = {} for suid, fle_list in fle_lsts.items(): suid = uuid.UUID(suid) sub = sub_objs[str(suid)] usid = uuid.UUID(sub_objs[str(suid)]['owner']) usr = usr_objs[str(usid)] auid = uuid.UUID(sub_objs[str(suid)]['assignment']) asn = asn_objs[str(auid)] if full_uuid: sub_str = "sub_{}".format(str(suid)) usr_str = "usr_{}".format(str(usid)) asn_str = "asn_{}".format(str(auid)) else: date = time.localtime(float(sub["created_time"])) date_str = time.strftime("%y%m%d_%H%M%S", date) sub_str = "sub_{}_{:012x}".format(date_str, suid.node) if full_name: full_nme = "".join("{}_{}".format(usr['last'], usr['first']).split()) usr_str = "usr_{}_{:012x}".format(full_nme, usid.node) else: user_nme = "".join(usr['username'].split()) usr_str = "usr_{}".format(user_nme) asgn_nme = "".join(asn['name'].split()) asn_str = "asn_{}_{:012x}".format(asgn_nme, auid.node) sub_path = os.path.join(dest_dir, asn_str, usr_str, sub_str) os.makedirs(sub_path, exist_ok=True) for fuid in fle_list: rel_path = fle_objs[fuid]["name"] rel_path = util_cli.clean_path(rel_path) rel_path = util_cli.secure_path(rel_path) fle_path = os.path.join(sub_path, rel_path) paths_map[fle_path] = fuid paths_set = set(paths_map.keys()) # Async Download Files def async_fun(path, paths_map): fuid = paths_map[path] return obj['files'].async_direct_download(fuid, path, overwrite=overwrite) label="Downloading Files " paths_out, paths_failed = async_obj_map(paths_set, async_fun, label=label, timing=timing, async_func_args=[paths_map]) # Display Errors: for puid, err in asn_lsts_failed: click.echo("Failed to list Assignments: {}".format(str(err))) for auid, err in asn_objs_failed.items(): click.echo("Failed to get Assignment '{}': {}".format(auid, str(err))) for auid, err in sub_lsts_failed.items(): click.echo("Failed to list Subs for Asn '{}': {}".format(audi, str(err))) for suid, err in sub_objs_failed.items(): click.echo("Failed to get Submission '{}': {}".format(suid, str(err))) for suid, err in fle_lsts_failed.items(): click.echo("Failed to list Files for Sub '{}': {}".format(suid, str(err))) for fuid, err in fle_objs_failed.items(): click.echo("Failed to get File '{}': {}".format(fuid, str(err))) for path, err in paths_failed.items(): basename = sys.path.basename(path) click.echo("Failed to download '{}': {}".format(basename, str(err))) # Display Stats: click.echo("Downloaded: {:6d} files".format(len(paths_out))) click.echo("Failed: {:6d} files".format(len(paths_failed))) if timing: end = time.time() dur = end - start dur_str = "Duration: {}".format(util_cli.duration_to_str(dur)) ops = len(paths_set)/dur ops_str = "Files/sec: {:11.2f}".format(ops) click.echo(dur_str) click.echo(ops_str)