def get_dir(self): res = [] self.cur_dir_meta = DirMetadata(self) response = self.s3.list_objects_v2(Bucket=self.bucket, Prefix=self.cur_dir_with_slash) for item in response.get('Contents', []): key = item['Key'] name = key.replace(self.cur_dir_with_slash, "") size = item['Size'] mtime = item['LastModified'].timestamp() etag = item['ETag'] # Current directory items only if (name.endswith("/") and len(name.split("/")) == 2) or (not (name.endswith("/")) and (not "/" in name)): if key.endswith("/") and size == 0: # "directory" res.append( DirectoryEntry(self, self.cur_dir, name.strip("/"), size, mtime, etag)) else: # "file" res.append( FileEntry(self, self.cur_dir, name, size, mtime, etag)) for item in response.get('Contents', []): key = item['Key'] name = key.replace(self.cur_dir_with_slash, "").split("/")[0] if not (name in [r.name for r in res]): res.append( DirectoryEntry(self, self.cur_dir, name.strip("/"), 0, 0, '')) return res
def get_dir(self): res = [] # self.cur_dir_meta = None self.cur_dir_meta = DirMetadata(self) for name in os.listdir(self.cur_dir): path = os.path.join(self.cur_dir, name) stat = os.lstat(path) # print(name) # print(" mt : %s" % stat.st_mtime) # print(" lc : %s" % (time.localtime(stat.st_mtime),)) # print(" : %s" % time.asctime(time.localtime(stat.st_mtime))) # print(" gmt: %s" % (time.gmtime(stat.st_mtime),)) # print(" : %s" % time.asctime(time.gmtime(stat.st_mtime))) # # utc_stamp = st_mtime_to_utc(stat.st_mtime) # print(" utc: %s" % utc_stamp) # print(" diff: %s" % ((utc_stamp - stat.st_mtime) / (60*60))) # stat.st_mtime is returned as UTC mtime = stat.st_mtime if os.path.isdir(path): res.append( DirectoryEntry(self, self.cur_dir, name, stat.st_size, mtime, str(stat.st_ino))) elif os.path.isfile(path): if name == DirMetadata.META_FILE_NAME: self.cur_dir_meta.read() elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ): res.append( FileEntry(self, self.cur_dir, name, stat.st_size, mtime, str(stat.st_ino))) return res
def _addline(line): data, _, name = line.partition("; ") res_type = size = mtime = unique = None fields = data.split(";") # http://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S.%f")) else: mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S")) # print("MLSD modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime)) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_res["has_meta"] = True elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir: # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ): entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: raise NotImplementedError("MLSD returned unsupported type: {!r}".format(res_type)) if entry: entry_map[name] = entry entry_list.append(entry) try: self.ftp.retrlines("MLSD", _addline) except error_perm as e: # print("The FTP server responded with {}".format(e), file=sys.stderr) # raises error_perm "500 Unknown command" if command is not supported if "500" in str(e.args): raise RuntimeError("The FTP server does not support the 'MLSD' command.") raise return
def _addline(line): data, _, name = line.partition("; ") res_type = size = mtime = unique = None fields = data.split(";") # http://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S")) # print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime)) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_res["has_meta"] = True elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ): entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: raise NotImplementedError if entry: entry_map[name] = entry entry_list.append(entry)
def get_dir(self): res = [] # self.cur_dir_meta = None self.cur_dir_meta = DirMetadata(self) # List directory. Pass in unicode on Py2, so we get unicode in return unicode_cur_dir = to_unicode(self.cur_dir) for name in os.listdir(unicode_cur_dir): name = to_native(name) path = os.path.join(self.cur_dir, name) stat = os.lstat(path) # write(name) # write(" mt : %s" % stat.st_mtime) # write(" lc : %s" % (time.localtime(stat.st_mtime),)) # write(" : %s" % time.asctime(time.localtime(stat.st_mtime))) # write(" gmt: %s" % (time.gmtime(stat.st_mtime),)) # write(" : %s" % time.asctime(time.gmtime(stat.st_mtime))) # utc_stamp = st_mtime_to_utc(stat.st_mtime) # write(" utc: %s" % utc_stamp) # write(" diff: %s" % ((utc_stamp - stat.st_mtime) / (60*60))) # stat.st_mtime is returned as UTC mtime = stat.st_mtime if os.path.isdir(path): res.append( DirectoryEntry(self, self.cur_dir, name, stat.st_size, mtime, str(stat.st_ino))) elif os.path.isfile(path): if name == DirMetadata.META_FILE_NAME: self.cur_dir_meta.read() # elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ): else: res.append( FileEntry( self, self.cur_dir, name, stat.st_size, mtime, str(stat.st_ino), )) return res
def _addline(status, line): # _ftp_retrlines_native() made sure that we always get `str` type lines assert status in (0, 1, 2) assert is_native(line) data, _, name = line.partition("; ") # print(status, name, u_name) if status == 1: write( "WARNING: File name seems not to be {}; re-encoded from CP-1252:" .format(encoding), name, ) elif status == 2: write_error("File name is neither UTF-8 nor CP-1252 encoded:", name) res_type = size = mtime = unique = None fields = data.split(";") # https://tools.ietf.org/html/rfc3659#page-23 # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang" # / "Media-Type" / "CharSet" / os-depend-fact / local-fact for field in fields: field_name, _, field_value = field.partition("=") field_name = field_name.lower() if field_name == "type": res_type = field_value elif field_name in ("sizd", "size"): size = int(field_value) elif field_name == "modify": # Use calendar.timegm() instead of time.mktime(), because # the date was returned as UTC if "." in field_value: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S.%f")) else: mtime = calendar.timegm( time.strptime(field_value, "%Y%m%d%H%M%S")) elif field_name == "unique": unique = field_value entry = None if res_type == "dir": entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type == "file": if name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored local_var["has_meta"] = True elif (name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir): # this is the root lock file. compare reported mtime with # local upload time self._probe_lock_file(mtime) else: entry = FileEntry(self, self.cur_dir, name, size, mtime, unique) elif res_type in ("cdir", "pdir"): pass else: write_error("Could not parse '{}'".format(line)) raise NotImplementedError( "MLSD returned unsupported type: {!r}".format(res_type)) if entry: entry_map[name] = entry entry_list.append(entry)
def _get_dir_impl(self): entry_list = [] entry_map = {} has_meta = False attr_list = self.sftp.listdir_attr() for de in attr_list: is_dir = stat.S_ISDIR(de.st_mode) name = de.filename entry = None if is_dir: if name not in (".", ".."): entry = DirectoryEntry( self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None ) elif name == DirMetadata.META_FILE_NAME: # the meta-data file is silently ignored has_meta = True elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir: # this is the root lock file. Compare reported mtime with # local upload time self._probe_lock_file(de.st_mtime) else: entry = FileEntry( self, self.cur_dir, name, de.st_size, de.st_mtime, unique=None ) if entry: entry_map[name] = entry entry_list.append(entry) # load stored meta data if present self.cur_dir_meta = DirMetadata(self) if has_meta: try: self.cur_dir_meta.read() except IncompatibleMetadataVersion: raise # this should end the script (user should pass --migrate) except Exception as e: write_error( "Could not read meta info {}: {}".format(self.cur_dir_meta, e) ) meta_files = self.cur_dir_meta.list # Adjust file mtime from meta-data if present missing = [] for n in meta_files: meta = meta_files[n] if n in entry_map: # We have a meta-data entry for this resource upload_time = meta.get("u", 0) # Discard stored meta-data if # 1. the reported files size is different than the # size we stored in the meta-data # or # 2. the the mtime reported by the SFTP server is later # than the stored upload time (which indicates # that the file was modified directly on the server) if entry_map[n].size != meta.get("s"): if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (size changed from {} to {}).".format( n, entry_map[n].size, meta.get("s") ) ) missing.append(n) elif (entry_map[n].mtime - upload_time) > self.mtime_compare_eps: if self.get_option("verbose", 3) >= 5: write( "Removing meta entry {} (modified {} > {}).".format( n, time.ctime(entry_map[n].mtime), time.ctime(upload_time), ) ) missing.append(n) else: # Use meta-data mtime instead of the one reported by SFTP server entry_map[n].meta = meta entry_map[n].mtime = meta["m"] else: # File is stored in meta-data, but no longer exists on SFTP server # write("META: Removing missing meta entry %s" % n) missing.append(n) # Remove missing or invalid files from cur_dir_meta for n in missing: self.cur_dir_meta.remove(n) # print("entry_list", entry_list) return entry_list