def process_dir(self, fitem, st): """ i_dir should be absolute path st is the stat object associated with the directory """ i_dir = fitem.path if self.dest: # we create destination directory # but first we check if we need to change mode for it to work o_dir = destpath(fitem, self.dest) mode = st.st_mode if not (st.st_mode & stat.S_IWUSR): mode = st.st_mode | stat.S_IWUSR self.opt_dir_list.append((o_dir, st)) try: os.mkdir(o_dir, mode) except OSError as e: log.debug("mkdir(): %s" % e, extra=self.d) if G.preserve: self.copy_xattr(i_dir, o_dir) last_report = MPI.Wtime() count = 0 try: entries = scandir(i_dir) except OSError as e: log.warn(e, extra=self.d) self.skipped += 1 else: for entry in entries: elefi = FileItem(entry.path) if fitem.dirname: elefi.dirname = fitem.dirname self.circle.enq(elefi) count += 1 if (MPI.Wtime() - last_report) > self.interval: print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, i_dir, count)) last_report = MPI.Wtime() log.info("Finish scan of [%s], count=%s" % (i_dir, count), extra=self.d)
def check_source_and_target(isrc, idest): """ verify and return target destination, isrc is iterable, idest is not. """ checked_src = [] checked_dup = set() is_dest_exist, is_dest_dir, is_dest_file, is_dest_parent_ok = False, False, False, False idest = os.path.abspath(idest) if os.path.exists(idest): if not os.access(idest, os.W_OK): err_and_exit("Destination is not accessible", 0) is_dest_exist = True if os.path.isfile(idest): is_dest_file = True elif os.path.isdir(idest): is_dest_dir = True else: # idest doesn't exits at this point # we check if its parent exists dest_parent = os.path.dirname(idest) if not (os.path.exists(dest_parent) and os.access(dest_parent, os.W_OK)): err_and_exit("Error: destination [%s] is not accessible" % dest_parent, 0) is_dest_parent_ok = True for ele in isrc: elepath = os.path.abspath(ele) elefi = FileItem(elepath) elefi.dirname = os.path.dirname(elepath) # save dirname for proper dest construction elebase = os.path.basename(elepath) if elebase in checked_dup: err_and_exit("Error: source name conflict detected: [%s]" % elepath) checked_dup.add(elebase) if os.path.exists(elepath) and os.access(elepath, os.R_OK): checked_src.append(elefi) else: err_and_exit("Error: source [%s] doesn't exist or not accessible." % ele, 0) if len(checked_src) == 0: err_and_exit("Error, no valid input", 0) elif len(checked_src) == 1 and os.path.isfile(checked_src[0].path): if is_dest_exist: if is_dest_file and args.force: try: os.remove(idest) except OSError as e: err_and_exit("Error: can't overwrite %s" % idest, 0) else: G.copytype = 'file2file' elif is_dest_dir: G.copytype = "file2dir" elif is_dest_parent_ok: G.copytype = 'file2file' else: err_and_exit("Error: can't detect correct copy type!", 0) elif len(checked_src) == 1 and not is_dest_exist: G.copytype = "dir2dir" elif len(checked_src) == 1 and is_dest_dir: if not args.force: err_and_exit("Error: destination [%s] exists, will not overwrite!" % idest) else: G.copytype = "dir2dir" else: # multiple sources, destination must be directory if not os.path.exists(idest): err_and_exit("Error: target directory %s doesn't exist!" % idest) if os.path.exists(idest) and os.path.isfile(idest): err_and_exit("Error: destination [%s] is a file, directory required" % idest, 0) # if is_dest_exist and not (args.force or args.rid): # err_and_exit("Destination [%s] exists, will not overwrite!" % idest, 0) G.copytype = "file2dir" return checked_src, idest