def _copy_recursive(self, src, dest, dir_entry):
# print("_copy_recursive(%s, %s --> %s)" % (dir_entry, src, dest))
assert isinstance(dir_entry, DirectoryEntry)
self._inc_stat("entries_touched")
self._inc_stat("dirs_created")
self._tick()
if self.dry_run:
return self._dry_run_action("copy directory (%s, %s --> %s)" %
(dir_entry, src, dest))
elif dest.readonly:
raise RuntimeError("target is read-only: %s" % dest)
dest.set_sync_info(dir_entry.name, None, None)
src.push_meta()
dest.push_meta()
src.cwd(dir_entry.name)
dest.mkdir(dir_entry.name)
dest.cwd(dir_entry.name)
dest.cur_dir_meta = DirMetadata(dest)
for entry in src.get_dir():
# the outer call was already accompanied by an increment, but not recursions
self._inc_stat("entries_seen")
if entry.is_dir():
self._copy_recursive(src, dest, entry)
else:
self._copy_file(src, dest, entry)
src.flush_meta()
dest.flush_meta()
src.cwd("..")
dest.cwd("..")
src.pop_meta()
dest.pop_meta()
return
def get_dir(self):
entry_list = []
entry_map = {}
local_res = {
"has_meta": False
} # pass local variables outside func scope
def _addline(line):
data, _, name = line.partition("; ")
res_type = size = mtime = unique = None
fields = data.split(";")
# http://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
# print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_res["has_meta"] = True
elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
raise NotImplementedError
if entry:
entry_map[name] = entry
entry_list.append(entry)
# raises error_perm, if command is not supported
self.ftp.retrlines("MLSD", _addline)
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if local_res["has_meta"]:
try:
self.cur_dir_meta.read()
except Exception as e:
print("Could not read meta info: %s" % e, file=sys.stderr)
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# TODO: use 3 sec EPS, to compare mtimes
if entry_map[n].size == meta.get(
"s") and entry_map[n].mtime <= upload_time:
# Use meta-data mtime instead of the one reported by FTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# Discard stored meta-data if
# 1. the the mtime reported by the FTP server is later
# than the stored upload time
# or
# 2. the reported files size is different than the
# size we stored in the meta-data
# print("META: Removing outdated meta entry %s" % n, meta)
missing.append(n)
else:
# File is stored in meta-data, but no longer exists on FTP server
# print("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
return entry_list