Esempio n. 1
0
    def process_dir(self, fitem, st):
        """ i_dir should be absolute path
        st is the stat object associated with the directory
        """
        i_dir = fitem.path

        if self.dest:
            # we create destination directory
            # but first we check if we need to change mode for it to work
            o_dir = destpath(fitem, self.dest)
            mode = st.st_mode
            if not (st.st_mode & stat.S_IWUSR):
                mode = st.st_mode | stat.S_IWUSR
                self.opt_dir_list.append((o_dir, st))
            try:
                os.mkdir(o_dir, mode)
            except OSError as e:
                log.debug("mkdir(): %s" % e, extra=self.d)

            if G.preserve:
                self.copy_xattr(i_dir, o_dir)

        last_report = MPI.Wtime()
        count = 0
        try:
            entries = scandir(i_dir)
        except OSError as e:
            log.warn(e, extra=self.d)
            self.skipped += 1
        else:
            for entry in entries:
                elefi = FileItem(entry.path)
                if fitem.dirname:
                    elefi.dirname = fitem.dirname
                self.circle.enq(elefi)

                count += 1
                if (MPI.Wtime() - last_report) > self.interval:
                    print("Rank %s : Scanning [%s] at %s" %
                          (self.circle.rank, i_dir, count))
                    last_report = MPI.Wtime()
            log.info("Finish scan of [%s], count=%s" % (i_dir, count),
                     extra=self.d)
Esempio n. 2
0
    def process_dir(self, fitem, st):
        """ i_dir should be absolute path
        st is the stat object associated with the directory
        """
        i_dir = fitem.path

        if self.dest:
            # we create destination directory
            # but first we check if we need to change mode for it to work
            o_dir = destpath(fitem, self.dest)
            mode = st.st_mode
            if not (st.st_mode & stat.S_IWUSR):
                mode = st.st_mode | stat.S_IWUSR
                self.opt_dir_list.append((o_dir, st))
            try:
                os.mkdir(o_dir, mode)
            except OSError as e:
                log.debug("mkdir(): %s" % e, extra=self.d)

            if G.preserve:
                self.copy_xattr(i_dir, o_dir)

        last_report = MPI.Wtime()
        count = 0
        try:
            entries = scandir(i_dir)
        except OSError as e:
            log.warn(e, extra=self.d)
            self.skipped += 1
        else:
            for entry in entries:
                elefi = FileItem(entry.path)
                if fitem.dirname:
                    elefi.dirname = fitem.dirname
                self.circle.enq(elefi)

                count += 1
                if (MPI.Wtime() - last_report) > self.interval:
                    print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, i_dir, count))
                    last_report = MPI.Wtime()
            log.info("Finish scan of [%s], count=%s" % (i_dir, count), extra=self.d)
Esempio n. 3
0
def check_source_and_target(isrc, idest):
    """ verify and return target destination, isrc is iterable, idest is not.
    """
    checked_src = []
    checked_dup = set()

    is_dest_exist, is_dest_dir, is_dest_file, is_dest_parent_ok = False, False, False, False

    idest = os.path.abspath(idest)

    if os.path.exists(idest):
        if not os.access(idest, os.W_OK):
            err_and_exit("Destination is not accessible", 0)
        is_dest_exist = True
        if os.path.isfile(idest):
            is_dest_file = True
        elif os.path.isdir(idest):
            is_dest_dir = True
    else:
        # idest doesn't exits at this point
        # we check if its parent exists
        dest_parent = os.path.dirname(idest)

        if not (os.path.exists(dest_parent) and os.access(dest_parent, os.W_OK)):
            err_and_exit("Error: destination [%s] is not accessible" % dest_parent, 0)
        is_dest_parent_ok = True

    for ele in isrc:
        elepath = os.path.abspath(ele)
        elefi = FileItem(elepath)
        elefi.dirname = os.path.dirname(elepath) # save dirname for proper dest construction
        elebase = os.path.basename(elepath)
        if elebase in checked_dup:
            err_and_exit("Error: source name conflict detected: [%s]" % elepath)
        checked_dup.add(elebase)

        if os.path.exists(elepath) and os.access(elepath, os.R_OK):
            checked_src.append(elefi)
        else:
            err_and_exit("Error: source [%s] doesn't exist or not accessible." % ele, 0)

    if len(checked_src) == 0:
        err_and_exit("Error, no valid input", 0)
    elif len(checked_src) == 1 and os.path.isfile(checked_src[0].path):
        if is_dest_exist:
            if is_dest_file and args.force:
                try:
                    os.remove(idest)
                except OSError as e:
                    err_and_exit("Error: can't overwrite %s" % idest, 0)
                else:
                    G.copytype = 'file2file'
            elif is_dest_dir:
                G.copytype = "file2dir"
        elif is_dest_parent_ok:
            G.copytype = 'file2file'
        else:
            err_and_exit("Error: can't detect correct copy type!", 0)
    elif len(checked_src) == 1 and not is_dest_exist:
        G.copytype = "dir2dir"
    elif len(checked_src) == 1 and is_dest_dir:
        if not args.force:
            err_and_exit("Error: destination [%s] exists, will not overwrite!" % idest)
        else:
            G.copytype = "dir2dir"
    else:
        # multiple sources, destination must be directory

        if not os.path.exists(idest):
            err_and_exit("Error: target directory %s doesn't exist!" % idest)

        if os.path.exists(idest) and os.path.isfile(idest):
            err_and_exit("Error: destination [%s] is a file, directory required" % idest, 0)

        # if is_dest_exist and not (args.force or args.rid):
        #     err_and_exit("Destination [%s] exists, will not overwrite!" % idest, 0)

        G.copytype = "file2dir"

    return checked_src, idest
Esempio n. 4
0
def check_source_and_target(isrc, idest):
    """ verify and return target destination, isrc is iterable, idest is not.
    """
    checked_src = []
    checked_dup = set()

    is_dest_exist, is_dest_dir, is_dest_file, is_dest_parent_ok = False, False, False, False

    idest = os.path.abspath(idest)

    if os.path.exists(idest):
        if not os.access(idest, os.W_OK):
            err_and_exit("Destination is not accessible", 0)
        is_dest_exist = True
        if os.path.isfile(idest):
            is_dest_file = True
        elif os.path.isdir(idest):
            is_dest_dir = True
    else:
        # idest doesn't exits at this point
        # we check if its parent exists
        dest_parent = os.path.dirname(idest)

        if not (os.path.exists(dest_parent) and os.access(dest_parent, os.W_OK)):
            err_and_exit("Error: destination [%s] is not accessible" % dest_parent, 0)
        is_dest_parent_ok = True

    for ele in isrc:
        elepath = os.path.abspath(ele)
        elefi = FileItem(elepath)
        elefi.dirname = os.path.dirname(elepath) # save dirname for proper dest construction
        elebase = os.path.basename(elepath)
        if elebase in checked_dup:
            err_and_exit("Error: source name conflict detected: [%s]" % elepath)
        checked_dup.add(elebase)

        if os.path.exists(elepath) and os.access(elepath, os.R_OK):
            checked_src.append(elefi)
        else:
            err_and_exit("Error: source [%s] doesn't exist or not accessible." % ele, 0)

    if len(checked_src) == 0:
        err_and_exit("Error, no valid input", 0)
    elif len(checked_src) == 1 and os.path.isfile(checked_src[0].path):
        if is_dest_exist:
            if is_dest_file and args.force:
                try:
                    os.remove(idest)
                except OSError as e:
                    err_and_exit("Error: can't overwrite %s" % idest, 0)
                else:
                    G.copytype = 'file2file'
            elif is_dest_dir:
                G.copytype = "file2dir"
        elif is_dest_parent_ok:
            G.copytype = 'file2file'
        else:
            err_and_exit("Error: can't detect correct copy type!", 0)
    elif len(checked_src) == 1 and not is_dest_exist:
        G.copytype = "dir2dir"
    elif len(checked_src) == 1 and is_dest_dir:
        if not args.force:
            err_and_exit("Error: destination [%s] exists, will not overwrite!" % idest)
        else:
            G.copytype = "dir2dir"
    else:
        # multiple sources, destination must be directory

        if not os.path.exists(idest):
            err_and_exit("Error: target directory %s doesn't exist!" % idest)

        if os.path.exists(idest) and os.path.isfile(idest):
            err_and_exit("Error: destination [%s] is a file, directory required" % idest, 0)

        # if is_dest_exist and not (args.force or args.rid):
        #     err_and_exit("Destination [%s] exists, will not overwrite!" % idest, 0)

        G.copytype = "file2dir"

    return checked_src, idest