Esempio n. 1
0
    def download(self, uid, path, orig_path=False, overwrite=False):

        # Clean Input
        path = os.path.abspath(path)

        # Process Directory Path
        if os.path.isdir(path):
            fle_obj = self.show(uid)
            fle_path = fle_obj["name"]
            fle_path = util_cli.clean_path(fle_path)
            fle_path = util_cli.secure_path(fle_path)
            fle_name = os.path.basename(fle_path)
            if orig_path:
                path = os.path.join(path, fle_path)
            else:
                path = os.path.join(path, fle_name)

        if overwrite or not os.path.exists(path):

            # Create Directory
            dir_path = os.path.dirname(path)
            os.makedirs(dir_path, exist_ok=True)

            # Download File
            ep = "{:s}/{:s}/{:s}/".format(self._ep, str(uid), _EP_FILES_CONTENTS)
            path = self._conn.http_download(ep, path)

        return path
Esempio n. 2
0
def util_download_submissions(obj, dest_dir, asn_list, sub_list, usr_list,
                              full_uuid, full_name, timing, overwrite):

    # Start Timing
    if timing:
        start = time.time()

    # Make Async Calls
    with obj['connection']:

        # Fetch Assignments
        tup = async_obj_fetch([None], obj_name="Assignments", timing=timing,
                              async_list=obj['assignments'].async_list_by_null,
                              async_show=obj['assignments'].async_show,
                              prefilter_list=asn_list)
        asn_lsts, asn_set, asn_objs, asn_lsts_failed, asn_objs_failed = tup

        # Fetch Submissions
        tup = async_obj_fetch(asn_set, obj_name="Submissions", timing=timing,
                              async_list=obj['submissions'].async_list_by_asn,
                              async_show=obj['submissions'].async_show,
                              prefilter_list=sub_list,
                              postfilter_func=postfilter_attr_owner,
                              postfilter_func_args=[usr_list])
        sub_lsts, sub_set, sub_objs, sub_lsts_failed, sub_objs_failed = tup

        # Fetch Files
        tup = async_obj_fetch(sub_set, obj_name="Files      ", timing=timing,
                              async_list=obj['files'].async_list_by_sub,
                              async_show=obj['files'].async_show)
        fle_lsts, fle_set, fle_objs, fle_lsts_failed, fle_objs_failed = tup

        # Fetch Users
        usr_set = set()
        for sub in sub_objs.values():
            usr_set.add(sub["owner"])
        usr_objs, usr_objs_failed = async_obj_map(usr_set, obj['users'].async_show,
                                                  label="Getting Users      ", timing=timing)

        # Build File Lists
        paths_map = {}
        for suid, fle_list in fle_lsts.items():

            suid = uuid.UUID(suid)
            sub = sub_objs[str(suid)]
            usid = uuid.UUID(sub_objs[str(suid)]['owner'])
            usr = usr_objs[str(usid)]
            auid = uuid.UUID(sub_objs[str(suid)]['assignment'])
            asn = asn_objs[str(auid)]

            if full_uuid:
                sub_str = "sub_{}".format(str(suid))
                usr_str = "usr_{}".format(str(usid))
                asn_str = "asn_{}".format(str(auid))
            else:
                date = time.localtime(float(sub["created_time"]))
                date_str = time.strftime("%y%m%d_%H%M%S", date)
                sub_str = "sub_{}_{:012x}".format(date_str, suid.node)
                if full_name:
                    full_nme = "".join("{}_{}".format(usr['last'], usr['first']).split())
                    usr_str = "usr_{}_{:012x}".format(full_nme, usid.node)
                else:
                    user_nme = "".join(usr['username'].split())
                    usr_str = "usr_{}".format(user_nme)
                asgn_nme = "".join(asn['name'].split())
                asn_str = "asn_{}_{:012x}".format(asgn_nme, auid.node)

            sub_path = os.path.join(dest_dir, asn_str, usr_str, sub_str)
            os.makedirs(sub_path, exist_ok=True)

            for fuid in fle_list:
                rel_path = fle_objs[fuid]["name"]
                rel_path = util_cli.clean_path(rel_path)
                rel_path = util_cli.secure_path(rel_path)
                fle_path = os.path.join(sub_path, rel_path)
                paths_map[fle_path] = fuid

        paths_set = set(paths_map.keys())

        # Async Download Files
        def async_fun(path, paths_map):
            fuid = paths_map[path]
            return obj['files'].async_direct_download(fuid, path, overwrite=overwrite)
        label="Downloading Files  "
        paths_out, paths_failed = async_obj_map(paths_set, async_fun,
                                                label=label, timing=timing,
                                                async_func_args=[paths_map])

    # Display Errors:
    for puid, err in asn_lsts_failed:
        click.echo("Failed to list Assignments: {}".format(str(err)))
    for auid, err in asn_objs_failed.items():
        click.echo("Failed to get Assignment '{}': {}".format(auid, str(err)))
    for auid, err in sub_lsts_failed.items():
        click.echo("Failed to list Subs for Asn '{}': {}".format(audi, str(err)))
    for suid, err in sub_objs_failed.items():
        click.echo("Failed to get Submission '{}': {}".format(suid, str(err)))
    for suid, err in fle_lsts_failed.items():
        click.echo("Failed to list Files for Sub '{}': {}".format(suid, str(err)))
    for fuid, err in fle_objs_failed.items():
        click.echo("Failed to get File '{}': {}".format(fuid, str(err)))
    for path, err in paths_failed.items():
        basename = sys.path.basename(path)
        click.echo("Failed to download '{}': {}".format(basename, str(err)))

    # Display Stats:
    click.echo("Downloaded: {:6d} files".format(len(paths_out)))
    click.echo("Failed:     {:6d} files".format(len(paths_failed)))
    if timing:
        end = time.time()
        dur = end - start
        dur_str = "Duration:    {}".format(util_cli.duration_to_str(dur))
        ops = len(paths_set)/dur
        ops_str = "Files/sec:   {:11.2f}".format(ops)
        click.echo(dur_str)
        click.echo(ops_str)