def handle_fitem(self, fi): if os.path.islink(fi.path): dest = destpath(fi, self.dest) linkto = os.readlink(fi.path) try: os.symlink(linkto, dest) except Exception as e: log.debug("%s, skipping sym link %s" % (e, fi.path), extra=self.d) elif stat.S_ISREG(fi.st_mode): self.enq_file(fi) # where chunking takes place
def process(self): """ process a work unit, spath, dpath refers to source and destination respectively """ fitem = self.circle.deq() spath = fitem.path if spath: try: st = os.lstat(spath) except OSError as e: log.warn(e, extra=self.d) self.skipped += 1 return False fitem.st_mode, fitem.st_size, fitem.st_uid, fitem.st_gid = st.st_mode, st.st_size, st.st_uid, st.st_gid self.reduce_items += 1 if os.path.islink(spath): self.append_fitem(fitem) self.sym_links += 1 # if not self.follow_sym_links: # NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT return if stat.S_ISREG(st.st_mode): if not self.dest: # fwalk without destination, simply add to process list self.append_fitem(fitem) else: # self.dest specified, need to check if it is there dpath = destpath(fitem, self.dest) flag = self.check_dest_exists(spath, dpath) if flag: return else: # if src and dest not the same # including the case dest is not there # then we do the following self.append_fitem(fitem) self.do_metadata_preserve(spath, dpath, st) self.cnt_files += 1 self.cnt_filesize += fitem.st_size elif stat.S_ISDIR(st.st_mode): self.cnt_dirs += 1 self.process_dir(fitem, st)
def process_dir(self, fitem, st): """ i_dir should be absolute path st is the stat object associated with the directory """ i_dir = fitem.path if self.dest: # we create destination directory # but first we check if we need to change mode for it to work o_dir = destpath(fitem, self.dest) mode = st.st_mode if not (st.st_mode & stat.S_IWUSR): mode = st.st_mode | stat.S_IWUSR self.opt_dir_list.append((o_dir, st)) try: os.mkdir(o_dir, mode) except OSError as e: log.debug("mkdir(): %s" % e, extra=self.d) if G.preserve: self.copy_xattr(i_dir, o_dir) last_report = MPI.Wtime() count = 0 try: entries = scandir(i_dir) except OSError as e: log.warn(e, extra=self.d) self.skipped += 1 else: for entry in entries: elefi = FileItem(entry.path) if fitem.dirname: elefi.dirname = fitem.dirname self.circle.enq(elefi) count += 1 if (MPI.Wtime() - last_report) > self.interval: print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, i_dir, count)) last_report = MPI.Wtime() log.info("Finish scan of [%s], count=%s" % (i_dir, count), extra=self.d)
def new_fchunk(self, fitem): fchunk = FileChunk() # default cmd = copy fchunk.src = fitem.path fchunk.dest = destpath(fitem, self.dest) return fchunk