def download(self, uid, path, orig_path=False, overwrite=False):
# Clean Input
path = os.path.abspath(path)
# Process Directory Path
if os.path.isdir(path):
fle_obj = self.show(uid)
fle_path = fle_obj["name"]
fle_path = util_cli.clean_path(fle_path)
fle_path = util_cli.secure_path(fle_path)
fle_name = os.path.basename(fle_path)
if orig_path:
path = os.path.join(path, fle_path)
else:
path = os.path.join(path, fle_name)
if overwrite or not os.path.exists(path):
# Create Directory
dir_path = os.path.dirname(path)
os.makedirs(dir_path, exist_ok=True)
# Download File
ep = "{:s}/{:s}/{:s}/".format(self._ep, str(uid), _EP_FILES_CONTENTS)
path = self._conn.http_download(ep, path)
return path
def util_download_submissions(obj, dest_dir, asn_list, sub_list, usr_list,
full_uuid, full_name, timing, overwrite):
# Start Timing
if timing:
start = time.time()
# Make Async Calls
with obj['connection']:
# Fetch Assignments
tup = async_obj_fetch([None], obj_name="Assignments", timing=timing,
async_list=obj['assignments'].async_list_by_null,
async_show=obj['assignments'].async_show,
prefilter_list=asn_list)
asn_lsts, asn_set, asn_objs, asn_lsts_failed, asn_objs_failed = tup
# Fetch Submissions
tup = async_obj_fetch(asn_set, obj_name="Submissions", timing=timing,
async_list=obj['submissions'].async_list_by_asn,
async_show=obj['submissions'].async_show,
prefilter_list=sub_list,
postfilter_func=postfilter_attr_owner,
postfilter_func_args=[usr_list])
sub_lsts, sub_set, sub_objs, sub_lsts_failed, sub_objs_failed = tup
# Fetch Files
tup = async_obj_fetch(sub_set, obj_name="Files ", timing=timing,
async_list=obj['files'].async_list_by_sub,
async_show=obj['files'].async_show)
fle_lsts, fle_set, fle_objs, fle_lsts_failed, fle_objs_failed = tup
# Fetch Users
usr_set = set()
for sub in sub_objs.values():
usr_set.add(sub["owner"])
usr_objs, usr_objs_failed = async_obj_map(usr_set, obj['users'].async_show,
label="Getting Users ", timing=timing)
# Build File Lists
paths_map = {}
for suid, fle_list in fle_lsts.items():
suid = uuid.UUID(suid)
sub = sub_objs[str(suid)]
usid = uuid.UUID(sub_objs[str(suid)]['owner'])
usr = usr_objs[str(usid)]
auid = uuid.UUID(sub_objs[str(suid)]['assignment'])
asn = asn_objs[str(auid)]
if full_uuid:
sub_str = "sub_{}".format(str(suid))
usr_str = "usr_{}".format(str(usid))
asn_str = "asn_{}".format(str(auid))
else:
date = time.localtime(float(sub["created_time"]))
date_str = time.strftime("%y%m%d_%H%M%S", date)
sub_str = "sub_{}_{:012x}".format(date_str, suid.node)
if full_name:
full_nme = "".join("{}_{}".format(usr['last'], usr['first']).split())
usr_str = "usr_{}_{:012x}".format(full_nme, usid.node)
else:
user_nme = "".join(usr['username'].split())
usr_str = "usr_{}".format(user_nme)
asgn_nme = "".join(asn['name'].split())
asn_str = "asn_{}_{:012x}".format(asgn_nme, auid.node)
sub_path = os.path.join(dest_dir, asn_str, usr_str, sub_str)
os.makedirs(sub_path, exist_ok=True)
for fuid in fle_list:
rel_path = fle_objs[fuid]["name"]
rel_path = util_cli.clean_path(rel_path)
rel_path = util_cli.secure_path(rel_path)
fle_path = os.path.join(sub_path, rel_path)
paths_map[fle_path] = fuid
paths_set = set(paths_map.keys())
# Async Download Files
def async_fun(path, paths_map):
fuid = paths_map[path]
return obj['files'].async_direct_download(fuid, path, overwrite=overwrite)
label="Downloading Files "
paths_out, paths_failed = async_obj_map(paths_set, async_fun,
label=label, timing=timing,
async_func_args=[paths_map])
# Display Errors:
for puid, err in asn_lsts_failed:
click.echo("Failed to list Assignments: {}".format(str(err)))
for auid, err in asn_objs_failed.items():
click.echo("Failed to get Assignment '{}': {}".format(auid, str(err)))
for auid, err in sub_lsts_failed.items():
click.echo("Failed to list Subs for Asn '{}': {}".format(audi, str(err)))
for suid, err in sub_objs_failed.items():
click.echo("Failed to get Submission '{}': {}".format(suid, str(err)))
for suid, err in fle_lsts_failed.items():
click.echo("Failed to list Files for Sub '{}': {}".format(suid, str(err)))
for fuid, err in fle_objs_failed.items():
click.echo("Failed to get File '{}': {}".format(fuid, str(err)))
for path, err in paths_failed.items():
basename = sys.path.basename(path)
click.echo("Failed to download '{}': {}".format(basename, str(err)))
# Display Stats:
click.echo("Downloaded: {:6d} files".format(len(paths_out)))
click.echo("Failed: {:6d} files".format(len(paths_failed)))
if timing:
end = time.time()
dur = end - start
dur_str = "Duration: {}".format(util_cli.duration_to_str(dur))
ops = len(paths_set)/dur
ops_str = "Files/sec: {:11.2f}".format(ops)
click.echo(dur_str)
click.echo(ops_str)
