def handle_fitem(self, fi):
if os.path.islink(fi.path):
dest = destpath(fi, self.dest)
linkto = os.readlink(fi.path)
try:
os.symlink(linkto, dest)
except Exception as e:
log.debug("%s, skipping sym link %s" % (e, fi.path), extra=self.d)
elif stat.S_ISREG(fi.st_mode):
self.enq_file(fi) # where chunking takes place
def handle_fitem(self, fi):
if os.path.islink(fi.path):
dest = destpath(fi, self.dest)
linkto = os.readlink(fi.path)
try:
os.symlink(linkto, dest)
except Exception as e:
log.debug("%s, skipping sym link %s" % (e, fi.path), extra=self.d)
elif stat.S_ISREG(fi.st_mode):
self.enq_file(fi) # where chunking takes place
def process(self):
""" process a work unit, spath, dpath refers to
source and destination respectively """
fitem = self.circle.deq()
spath = fitem.path
if spath:
try:
st = os.lstat(spath)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
return False
fitem.st_mode, fitem.st_size, fitem.st_uid, fitem.st_gid = st.st_mode, st.st_size, st.st_uid, st.st_gid
self.reduce_items += 1
if os.path.islink(spath):
self.append_fitem(fitem)
self.sym_links += 1
# if not self.follow_sym_links:
# NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT
return
if stat.S_ISREG(st.st_mode):
if not self.dest:
# fwalk without destination, simply add to process list
self.append_fitem(fitem)
else:
# self.dest specified, need to check if it is there
dpath = destpath(fitem, self.dest)
flag = self.check_dest_exists(spath, dpath)
if flag:
return
else:
# if src and dest not the same
# including the case dest is not there
# then we do the following
self.append_fitem(fitem)
self.do_metadata_preserve(spath, dpath, st)
self.cnt_files += 1
self.cnt_filesize += fitem.st_size
elif stat.S_ISDIR(st.st_mode):
self.cnt_dirs += 1
self.process_dir(fitem, st)
def process(self):
""" process a work unit, spath, dpath refers to
source and destination respectively """
fitem = self.circle.deq()
spath = fitem.path
if spath:
try:
st = os.lstat(spath)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
return False
fitem.st_mode, fitem.st_size, fitem.st_uid, fitem.st_gid = st.st_mode, st.st_size, st.st_uid, st.st_gid
self.reduce_items += 1
if os.path.islink(spath):
self.append_fitem(fitem)
self.sym_links += 1
# if not self.follow_sym_links:
# NOT TO FOLLOW SYM LINKS SHOULD BE THE DEFAULT
return
if stat.S_ISREG(st.st_mode):
if not self.dest:
# fwalk without destination, simply add to process list
self.append_fitem(fitem)
else:
# self.dest specified, need to check if it is there
dpath = destpath(fitem, self.dest)
flag = self.check_dest_exists(spath, dpath)
if flag:
return
else:
# if src and dest not the same
# including the case dest is not there
# then we do the following
self.append_fitem(fitem)
self.do_metadata_preserve(spath, dpath, st)
self.cnt_files += 1
self.cnt_filesize += fitem.st_size
elif stat.S_ISDIR(st.st_mode):
self.cnt_dirs += 1
self.process_dir(fitem, st)
def process_dir(self, fitem, st):
""" i_dir should be absolute path
st is the stat object associated with the directory
"""
i_dir = fitem.path
if self.dest:
# we create destination directory
# but first we check if we need to change mode for it to work
o_dir = destpath(fitem, self.dest)
mode = st.st_mode
if not (st.st_mode & stat.S_IWUSR):
mode = st.st_mode | stat.S_IWUSR
self.opt_dir_list.append((o_dir, st))
try:
os.mkdir(o_dir, mode)
except OSError as e:
log.debug("mkdir(): %s" % e, extra=self.d)
if G.preserve:
self.copy_xattr(i_dir, o_dir)
last_report = MPI.Wtime()
count = 0
try:
entries = scandir(i_dir)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
else:
for entry in entries:
elefi = FileItem(entry.path)
if fitem.dirname:
elefi.dirname = fitem.dirname
self.circle.enq(elefi)
count += 1
if (MPI.Wtime() - last_report) > self.interval:
print("Rank %s : Scanning [%s] at %s" %
(self.circle.rank, i_dir, count))
last_report = MPI.Wtime()
log.info("Finish scan of [%s], count=%s" % (i_dir, count),
extra=self.d)
def process_dir(self, fitem, st):
""" i_dir should be absolute path
st is the stat object associated with the directory
"""
i_dir = fitem.path
if self.dest:
# we create destination directory
# but first we check if we need to change mode for it to work
o_dir = destpath(fitem, self.dest)
mode = st.st_mode
if not (st.st_mode & stat.S_IWUSR):
mode = st.st_mode | stat.S_IWUSR
self.opt_dir_list.append((o_dir, st))
try:
os.mkdir(o_dir, mode)
except OSError as e:
log.debug("mkdir(): %s" % e, extra=self.d)
if G.preserve:
self.copy_xattr(i_dir, o_dir)
last_report = MPI.Wtime()
count = 0
try:
entries = scandir(i_dir)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
else:
for entry in entries:
elefi = FileItem(entry.path)
if fitem.dirname:
elefi.dirname = fitem.dirname
self.circle.enq(elefi)
count += 1
if (MPI.Wtime() - last_report) > self.interval:
print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, i_dir, count))
last_report = MPI.Wtime()
log.info("Finish scan of [%s], count=%s" % (i_dir, count), extra=self.d)
def new_fchunk(self, fitem):
fchunk = FileChunk() # default cmd = copy
fchunk.src = fitem.path
fchunk.dest = destpath(fitem, self.dest)
return fchunk
def new_fchunk(self, fitem):
fchunk = FileChunk() # default cmd = copy
fchunk.src = fitem.path
fchunk.dest = destpath(fitem, self.dest)
return fchunk
