def get_dir(self):
res = []
self.cur_dir_meta = DirMetadata(self)
response = self.s3.list_objects_v2(Bucket=self.bucket,
Prefix=self.cur_dir_with_slash)
for item in response.get('Contents', []):
key = item['Key']
name = key.replace(self.cur_dir_with_slash, "")
size = item['Size']
mtime = item['LastModified'].timestamp()
etag = item['ETag']
# Current directory items only
if (name.endswith("/")
and len(name.split("/")) == 2) or (not (name.endswith("/"))
and (not "/" in name)):
if key.endswith("/") and size == 0: # "directory"
res.append(
DirectoryEntry(self, self.cur_dir, name.strip("/"),
size, mtime, etag))
else: # "file"
res.append(
FileEntry(self, self.cur_dir, name, size, mtime, etag))
for item in response.get('Contents', []):
key = item['Key']
name = key.replace(self.cur_dir_with_slash, "").split("/")[0]
if not (name in [r.name for r in res]):
res.append(
DirectoryEntry(self, self.cur_dir, name.strip("/"), 0, 0,
''))
return res
def get_dir(self):
res = []
# self.cur_dir_meta = None
self.cur_dir_meta = DirMetadata(self)
for name in os.listdir(self.cur_dir):
path = os.path.join(self.cur_dir, name)
stat = os.lstat(path)
# print(name)
# print(" mt : %s" % stat.st_mtime)
# print(" lc : %s" % (time.localtime(stat.st_mtime),))
# print(" : %s" % time.asctime(time.localtime(stat.st_mtime)))
# print(" gmt: %s" % (time.gmtime(stat.st_mtime),))
# print(" : %s" % time.asctime(time.gmtime(stat.st_mtime)))
#
# utc_stamp = st_mtime_to_utc(stat.st_mtime)
# print(" utc: %s" % utc_stamp)
# print(" diff: %s" % ((utc_stamp - stat.st_mtime) / (60*60)))
# stat.st_mtime is returned as UTC
mtime = stat.st_mtime
if os.path.isdir(path):
res.append(
DirectoryEntry(self, self.cur_dir, name, stat.st_size,
mtime, str(stat.st_ino)))
elif os.path.isfile(path):
if name == DirMetadata.META_FILE_NAME:
self.cur_dir_meta.read()
elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
res.append(
FileEntry(self, self.cur_dir, name, stat.st_size,
mtime, str(stat.st_ino)))
return res
def _addline(line):
data, _, name = line.partition("; ")
res_type = size = mtime = unique = None
fields = data.split(";")
# http://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S"))
# print("MLSD modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_res["has_meta"] = True
elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
elif res_type in ("cdir", "pdir"):
pass
else:
raise NotImplementedError("MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
try:
self.ftp.retrlines("MLSD", _addline)
except error_perm as e:
# print("The FTP server responded with {}".format(e), file=sys.stderr)
# raises error_perm "500 Unknown command" if command is not supported
if "500" in str(e.args):
raise RuntimeError("The FTP server does not support the 'MLSD' command.")
raise
return
def _addline(line):
data, _, name = line.partition("; ")
res_type = size = mtime = unique = None
fields = data.split(";")
# http://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
# print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_res["has_meta"] = True
elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
raise NotImplementedError
if entry:
entry_map[name] = entry
entry_list.append(entry)
def get_dir(self):
res = []
# self.cur_dir_meta = None
self.cur_dir_meta = DirMetadata(self)
# List directory. Pass in unicode on Py2, so we get unicode in return
unicode_cur_dir = to_unicode(self.cur_dir)
for name in os.listdir(unicode_cur_dir):
name = to_native(name)
path = os.path.join(self.cur_dir, name)
stat = os.lstat(path)
# write(name)
# write(" mt : %s" % stat.st_mtime)
# write(" lc : %s" % (time.localtime(stat.st_mtime),))
# write(" : %s" % time.asctime(time.localtime(stat.st_mtime)))
# write(" gmt: %s" % (time.gmtime(stat.st_mtime),))
# write(" : %s" % time.asctime(time.gmtime(stat.st_mtime)))
# utc_stamp = st_mtime_to_utc(stat.st_mtime)
# write(" utc: %s" % utc_stamp)
# write(" diff: %s" % ((utc_stamp - stat.st_mtime) / (60*60)))
# stat.st_mtime is returned as UTC
mtime = stat.st_mtime
if os.path.isdir(path):
res.append(
DirectoryEntry(self, self.cur_dir, name, stat.st_size,
mtime, str(stat.st_ino)))
elif os.path.isfile(path):
if name == DirMetadata.META_FILE_NAME:
self.cur_dir_meta.read()
# elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
else:
res.append(
FileEntry(
self,
self.cur_dir,
name,
stat.st_size,
mtime,
str(stat.st_ino),
))
return res
def _addline(status, line):
# _ftp_retrlines_native() made sure that we always get `str` type lines
assert status in (0, 1, 2)
assert is_native(line)
data, _, name = line.partition("; ")
# print(status, name, u_name)
if status == 1:
write(
"WARNING: File name seems not to be {}; re-encoded from CP-1252:"
.format(encoding),
name,
)
elif status == 2:
write_error("File name is neither UTF-8 nor CP-1252 encoded:",
name)
res_type = size = mtime = unique = None
fields = data.split(";")
# https://tools.ietf.org/html/rfc3659#page-23
# "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
# / "Media-Type" / "CharSet" / os-depend-fact / local-fact
for field in fields:
field_name, _, field_value = field.partition("=")
field_name = field_name.lower()
if field_name == "type":
res_type = field_value
elif field_name in ("sizd", "size"):
size = int(field_value)
elif field_name == "modify":
# Use calendar.timegm() instead of time.mktime(), because
# the date was returned as UTC
if "." in field_value:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
else:
mtime = calendar.timegm(
time.strptime(field_value, "%Y%m%d%H%M%S"))
elif field_name == "unique":
unique = field_value
entry = None
if res_type == "dir":
entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type == "file":
if name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
local_var["has_meta"] = True
elif (name == DirMetadata.LOCK_FILE_NAME
and self.cur_dir == self.root_dir):
# this is the root lock file. compare reported mtime with
# local upload time
self._probe_lock_file(mtime)
else:
entry = FileEntry(self, self.cur_dir, name, size, mtime,
unique)
elif res_type in ("cdir", "pdir"):
pass
else:
write_error("Could not parse '{}'".format(line))
raise NotImplementedError(
"MLSD returned unsupported type: {!r}".format(res_type))
if entry:
entry_map[name] = entry
entry_list.append(entry)
def _get_dir_impl(self):
entry_list = []
entry_map = {}
has_meta = False
attr_list = self.sftp.listdir_attr()
for de in attr_list:
is_dir = stat.S_ISDIR(de.st_mode)
name = de.filename
entry = None
if is_dir:
if name not in (".", ".."):
entry = DirectoryEntry(
self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None
)
elif name == DirMetadata.META_FILE_NAME:
# the meta-data file is silently ignored
has_meta = True
elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
# this is the root lock file. Compare reported mtime with
# local upload time
self._probe_lock_file(de.st_mtime)
else:
entry = FileEntry(
self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None
)
if entry:
entry_map[name] = entry
entry_list.append(entry)
# load stored meta data if present
self.cur_dir_meta = DirMetadata(self)
if has_meta:
try:
self.cur_dir_meta.read()
except IncompatibleMetadataVersion:
raise # this should end the script (user should pass --migrate)
except Exception as e:
write_error(
"Could not read meta info {}: {}".format(self.cur_dir_meta, e)
)
meta_files = self.cur_dir_meta.list
# Adjust file mtime from meta-data if present
missing = []
for n in meta_files:
meta = meta_files[n]
if n in entry_map:
# We have a meta-data entry for this resource
upload_time = meta.get("u", 0)
# Discard stored meta-data if
# 1. the reported files size is different than the
# size we stored in the meta-data
# or
# 2. the the mtime reported by the SFTP server is later
# than the stored upload time (which indicates
# that the file was modified directly on the server)
if entry_map[n].size != meta.get("s"):
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (size changed from {} to {}).".format(
n, entry_map[n].size, meta.get("s")
)
)
missing.append(n)
elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps:
if self.get_option("verbose", 3) >= 5:
write(
"Removing meta entry {} (modified {} > {}).".format(
n,
time.ctime(entry_map[n].mtime),
time.ctime(upload_time),
)
)
missing.append(n)
else:
# Use meta-data mtime instead of the one reported by SFTP server
entry_map[n].meta = meta
entry_map[n].mtime = meta["m"]
else:
# File is stored in meta-data, but no longer exists on SFTP server
# write("META: Removing missing meta entry %s" % n)
missing.append(n)
# Remove missing or invalid files from cur_dir_meta
for n in missing:
self.cur_dir_meta.remove(n)
# print("entry_list", entry_list)
return entry_list