def process_dir(self, fitem, st):
""" i_dir should be absolute path
st is the stat object associated with the directory
"""
i_dir = fitem.path
if self.dest:
# we create destination directory
# but first we check if we need to change mode for it to work
o_dir = destpath(fitem, self.dest)
mode = st.st_mode
if not (st.st_mode & stat.S_IWUSR):
mode = st.st_mode | stat.S_IWUSR
self.opt_dir_list.append((o_dir, st))
try:
os.mkdir(o_dir, mode)
except OSError as e:
log.debug("mkdir(): %s" % e, extra=self.d)
if G.preserve:
self.copy_xattr(i_dir, o_dir)
last_report = MPI.Wtime()
count = 0
try:
entries = scandir(i_dir)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
else:
for entry in entries:
elefi = FileItem(entry.path)
if fitem.dirname:
elefi.dirname = fitem.dirname
self.circle.enq(elefi)
count += 1
if (MPI.Wtime() - last_report) > self.interval:
print("Rank %s : Scanning [%s] at %s" %
(self.circle.rank, i_dir, count))
last_report = MPI.Wtime()
log.info("Finish scan of [%s], count=%s" % (i_dir, count),
extra=self.d)
def process_dir(self, fitem, st):
""" i_dir should be absolute path
st is the stat object associated with the directory
"""
i_dir = fitem.path
if self.dest:
# we create destination directory
# but first we check if we need to change mode for it to work
o_dir = destpath(fitem, self.dest)
mode = st.st_mode
if not (st.st_mode & stat.S_IWUSR):
mode = st.st_mode | stat.S_IWUSR
self.opt_dir_list.append((o_dir, st))
try:
os.mkdir(o_dir, mode)
except OSError as e:
log.debug("mkdir(): %s" % e, extra=self.d)
if G.preserve:
self.copy_xattr(i_dir, o_dir)
last_report = MPI.Wtime()
count = 0
try:
entries = scandir(i_dir)
except OSError as e:
log.warn(e, extra=self.d)
self.skipped += 1
else:
for entry in entries:
elefi = FileItem(entry.path)
if fitem.dirname:
elefi.dirname = fitem.dirname
self.circle.enq(elefi)
count += 1
if (MPI.Wtime() - last_report) > self.interval:
print("Rank %s : Scanning [%s] at %s" % (self.circle.rank, i_dir, count))
last_report = MPI.Wtime()
log.info("Finish scan of [%s], count=%s" % (i_dir, count), extra=self.d)
def check_source_and_target(isrc, idest):
""" verify and return target destination, isrc is iterable, idest is not.
"""
checked_src = []
checked_dup = set()
is_dest_exist, is_dest_dir, is_dest_file, is_dest_parent_ok = False, False, False, False
idest = os.path.abspath(idest)
if os.path.exists(idest):
if not os.access(idest, os.W_OK):
err_and_exit("Destination is not accessible", 0)
is_dest_exist = True
if os.path.isfile(idest):
is_dest_file = True
elif os.path.isdir(idest):
is_dest_dir = True
else:
# idest doesn't exits at this point
# we check if its parent exists
dest_parent = os.path.dirname(idest)
if not (os.path.exists(dest_parent) and os.access(dest_parent, os.W_OK)):
err_and_exit("Error: destination [%s] is not accessible" % dest_parent, 0)
is_dest_parent_ok = True
for ele in isrc:
elepath = os.path.abspath(ele)
elefi = FileItem(elepath)
elefi.dirname = os.path.dirname(elepath) # save dirname for proper dest construction
elebase = os.path.basename(elepath)
if elebase in checked_dup:
err_and_exit("Error: source name conflict detected: [%s]" % elepath)
checked_dup.add(elebase)
if os.path.exists(elepath) and os.access(elepath, os.R_OK):
checked_src.append(elefi)
else:
err_and_exit("Error: source [%s] doesn't exist or not accessible." % ele, 0)
if len(checked_src) == 0:
err_and_exit("Error, no valid input", 0)
elif len(checked_src) == 1 and os.path.isfile(checked_src[0].path):
if is_dest_exist:
if is_dest_file and args.force:
try:
os.remove(idest)
except OSError as e:
err_and_exit("Error: can't overwrite %s" % idest, 0)
else:
G.copytype = 'file2file'
elif is_dest_dir:
G.copytype = "file2dir"
elif is_dest_parent_ok:
G.copytype = 'file2file'
else:
err_and_exit("Error: can't detect correct copy type!", 0)
elif len(checked_src) == 1 and not is_dest_exist:
G.copytype = "dir2dir"
elif len(checked_src) == 1 and is_dest_dir:
if not args.force:
err_and_exit("Error: destination [%s] exists, will not overwrite!" % idest)
else:
G.copytype = "dir2dir"
else:
# multiple sources, destination must be directory
if not os.path.exists(idest):
err_and_exit("Error: target directory %s doesn't exist!" % idest)
if os.path.exists(idest) and os.path.isfile(idest):
err_and_exit("Error: destination [%s] is a file, directory required" % idest, 0)
# if is_dest_exist and not (args.force or args.rid):
# err_and_exit("Destination [%s] exists, will not overwrite!" % idest, 0)
G.copytype = "file2dir"
return checked_src, idest
def check_source_and_target(isrc, idest):
""" verify and return target destination, isrc is iterable, idest is not.
"""
checked_src = []
checked_dup = set()
is_dest_exist, is_dest_dir, is_dest_file, is_dest_parent_ok = False, False, False, False
idest = os.path.abspath(idest)
if os.path.exists(idest):
if not os.access(idest, os.W_OK):
err_and_exit("Destination is not accessible", 0)
is_dest_exist = True
if os.path.isfile(idest):
is_dest_file = True
elif os.path.isdir(idest):
is_dest_dir = True
else:
# idest doesn't exits at this point
# we check if its parent exists
dest_parent = os.path.dirname(idest)
if not (os.path.exists(dest_parent) and os.access(dest_parent, os.W_OK)):
err_and_exit("Error: destination [%s] is not accessible" % dest_parent, 0)
is_dest_parent_ok = True
for ele in isrc:
elepath = os.path.abspath(ele)
elefi = FileItem(elepath)
elefi.dirname = os.path.dirname(elepath) # save dirname for proper dest construction
elebase = os.path.basename(elepath)
if elebase in checked_dup:
err_and_exit("Error: source name conflict detected: [%s]" % elepath)
checked_dup.add(elebase)
if os.path.exists(elepath) and os.access(elepath, os.R_OK):
checked_src.append(elefi)
else:
err_and_exit("Error: source [%s] doesn't exist or not accessible." % ele, 0)
if len(checked_src) == 0:
err_and_exit("Error, no valid input", 0)
elif len(checked_src) == 1 and os.path.isfile(checked_src[0].path):
if is_dest_exist:
if is_dest_file and args.force:
try:
os.remove(idest)
except OSError as e:
err_and_exit("Error: can't overwrite %s" % idest, 0)
else:
G.copytype = 'file2file'
elif is_dest_dir:
G.copytype = "file2dir"
elif is_dest_parent_ok:
G.copytype = 'file2file'
else:
err_and_exit("Error: can't detect correct copy type!", 0)
elif len(checked_src) == 1 and not is_dest_exist:
G.copytype = "dir2dir"
elif len(checked_src) == 1 and is_dest_dir:
if not args.force:
err_and_exit("Error: destination [%s] exists, will not overwrite!" % idest)
else:
G.copytype = "dir2dir"
else:
# multiple sources, destination must be directory
if not os.path.exists(idest):
err_and_exit("Error: target directory %s doesn't exist!" % idest)
if os.path.exists(idest) and os.path.isfile(idest):
err_and_exit("Error: destination [%s] is a file, directory required" % idest, 0)
# if is_dest_exist and not (args.force or args.rid):
# err_and_exit("Destination [%s] exists, will not overwrite!" % idest, 0)
G.copytype = "file2dir"
return checked_src, idest