Exemplo n.º 1
0
    def _copy_recursive(self, src, dest, dir_entry):
        #        print("_copy_recursive(%s, %s --> %s)" % (dir_entry, src, dest))
        assert isinstance(dir_entry, DirectoryEntry)
        self._inc_stat("entries_touched")
        self._inc_stat("dirs_created")
        self._tick()
        if self.dry_run:
            return self._dry_run_action("copy directory (%s, %s --> %s)" %
                                        (dir_entry, src, dest))
        elif dest.readonly:
            raise RuntimeError("target is read-only: %s" % dest)

        dest.set_sync_info(dir_entry.name, None, None)

        src.push_meta()
        dest.push_meta()

        src.cwd(dir_entry.name)
        dest.mkdir(dir_entry.name)
        dest.cwd(dir_entry.name)
        dest.cur_dir_meta = DirMetadata(dest)
        for entry in src.get_dir():
            # the outer call was already accompanied by an increment, but not recursions
            self._inc_stat("entries_seen")
            if entry.is_dir():
                self._copy_recursive(src, dest, entry)
            else:
                self._copy_file(src, dest, entry)

        src.flush_meta()
        dest.flush_meta()

        src.cwd("..")
        dest.cwd("..")

        src.pop_meta()
        dest.pop_meta()
        return
Exemplo n.º 2
0
class FtpTarget(_Target):
    def __init__(self,
                 path,
                 host,
                 port,
                 username=None,
                 password=None,
                 extra_opts=None):
        path = path or "/"
        super(FtpTarget, self).__init__(path, extra_opts)
        self.ftp = ftplib.FTP()
        self.ftp.debug(self.get_option("ftp_debug", 0))
        self.host = host
        self.port = port
        self.username = username
        self.password = password
#        if connect:
#            self.open()

    def __str__(self):
        return "<ftp:%s%s + %s>" % (self.host, self.root_dir,
                                    relpath_url(self.cur_dir, self.root_dir))

    def get_base_name(self):
        return "ftp:%s%s" % (self.host, self.root_dir)

    def open(self):
        assert not self.connected
        no_prompt = self.get_option("no_prompt", True)
        store_password = self.get_option("store_password", False)

        if self.port:
            self.ftp.connect(self.host, self.port)
        else:
            self.ftp.connect(self.host)
        #
        if self.username is None or self.password is None:
            creds = get_credentials_for_url(self.host,
                                            allow_prompt=not no_prompt)
            if creds:
                self.username, self.password = creds

        try:
            # Login (as 'anonymous' if self.username is undefined):
            self.ftp.login(self.username, self.password)
        except error_perm as e:
            # If credentials were passed, but authentication fails, prompt
            # for new password
            if not e.args[0].startswith("530"):
                raise  # error other then '530 Login incorrect'
            print(e)
            if not no_prompt:
                self.user, self.password = prompt_for_password(
                    self.host, self.username)
                self.ftp.login(self.username, self.password)

        # TODO: case sensitivity?
#        resp = self.ftp.sendcmd("system")
#        self.is_unix = "unix" in resp.lower() # not necessarily true, better check with read/write tests

        try:
            #
            self.ftp.cwd(self.root_dir)
        except error_perm as e:
            # If credentials were passed, but authentication fails, prompt
            # for new password
            if not e.args[0].startswith("550"):
                raise  # error other then 550 No such directory'
            print(
                "Could not change directory to %s (%s): missing permissions?" %
                (self.root_dir, e))

        pwd = self.ftp.pwd()
        if pwd != self.root_dir:
            raise RuntimeError("Unable to navigate to working directory %r" %
                               self.root_dir)
        self.cur_dir = pwd
        self.connected = True
        # Successfully authenticated: store password
        if store_password:
            save_password(self.host, self.username, self.password)
        return

    def close(self):
        if self.connected:
            self.ftp.quit()
        self.connected = False

    def get_id(self):
        return self.host + self.root_dir

    def cwd(self, dir_name):
        path = normpath_url(join_url(self.cur_dir, dir_name))
        if not path.startswith(self.root_dir):
            # paranoic check to prevent that our sync tool goes berserk
            raise RuntimeError("Tried to navigate outside root %r: %r" %
                               (self.root_dir, path))
        self.ftp.cwd(dir_name)
        self.cur_dir = path
        self.cur_dir_meta = None
        return self.cur_dir

    def pwd(self):
        return self.ftp.pwd()

    def mkdir(self, dir_name):
        self.check_write(dir_name)
        self.ftp.mkd(dir_name)

    def _rmdir_impl(self, dir_name, keep_root=False):
        # FTP does not support deletion of non-empty directories.
        #        print("rmdir(%s)" % dir_name)
        self.check_write(dir_name)
        names = self.ftp.nlst(dir_name)
        #        print("rmdir(%s): %s" % (dir_name, names))
        # Skip ftp.cwd(), if dir is empty
        names = [n for n in names if n not in (".", "..")]
        if len(names) > 0:
            self.ftp.cwd(dir_name)
            try:
                for name in names:
                    try:
                        # try to delete this as a file
                        self.ftp.delete(name)
                    except ftplib.all_errors as _e:
                        #                        print("    ftp.delete(%s) failed: %s, trying rmdir()..." % (name, _e))
                        # assume <name> is a folder
                        self.rmdir(name)
            finally:
                if dir_name != ".":
                    self.ftp.cwd("..")
#        print("ftp.rmd(%s)..." % (dir_name, ))
        if not keep_root:
            self.ftp.rmd(dir_name)
        return

    def rmdir(self, dir_name):
        return self._rmdir_impl(dir_name)

    def get_dir(self):
        entry_list = []
        entry_map = {}
        local_res = {
            "has_meta": False
        }  # pass local variables outside func scope

        def _addline(line):
            data, _, name = line.partition("; ")
            res_type = size = mtime = unique = None
            fields = data.split(";")
            # http://tools.ietf.org/html/rfc3659#page-23
            # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
            #   / "Media-Type" / "CharSet" / os-depend-fact / local-fact
            for field in fields:
                field_name, _, field_value = field.partition("=")
                field_name = field_name.lower()
                if field_name == "type":
                    res_type = field_value
                elif field_name in ("sizd", "size"):
                    size = int(field_value)
                elif field_name == "modify":
                    # Use calendar.timegm() instead of time.mktime(), because
                    # the date was returned as UTC
                    mtime = calendar.timegm(
                        time.strptime(field_value, "%Y%m%d%H%M%S"))
#                    print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
                elif field_name == "unique":
                    unique = field_value

            entry = None
            if res_type == "dir":
                entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
                                       unique)
            elif res_type == "file":
                if name == DirMetadata.META_FILE_NAME:
                    # the meta-data file is silently ignored
                    local_res["has_meta"] = True
                elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
                    entry = FileEntry(self, self.cur_dir, name, size, mtime,
                                      unique)
            elif res_type in ("cdir", "pdir"):
                pass
            else:
                raise NotImplementedError

            if entry:
                entry_map[name] = entry
                entry_list.append(entry)

        # raises error_perm, if command is not supported
        self.ftp.retrlines("MLSD", _addline)

        # load stored meta data if present
        self.cur_dir_meta = DirMetadata(self)

        if local_res["has_meta"]:
            try:
                self.cur_dir_meta.read()
            except Exception as e:
                print("Could not read meta info: %s" % e, file=sys.stderr)

            meta_files = self.cur_dir_meta.list

            # Adjust file mtime from meta-data if present
            missing = []
            for n in meta_files:
                meta = meta_files[n]
                if n in entry_map:
                    # We have a meta-data entry for this resource
                    upload_time = meta.get("u", 0)
                    # TODO: use 3 sec EPS, to compare mtimes
                    if entry_map[n].size == meta.get(
                            "s") and entry_map[n].mtime <= upload_time:
                        # Use meta-data mtime instead of the one reported by FTP server
                        entry_map[n].meta = meta
                        entry_map[n].mtime = meta["m"]
                    else:
                        # Discard stored meta-data if
                        #   1. the the mtime reported by the FTP server is later
                        #      than the stored upload time
                        #      or
                        #   2. the reported files size is different than the
                        #      size we stored in the meta-data
                        #                        print("META: Removing outdated meta entry %s" % n, meta)
                        missing.append(n)
                else:
                    # File is stored in meta-data, but no longer exists on FTP server
                    #                     print("META: Removing missing meta entry %s" % n)
                    missing.append(n)
            # Remove missing or invalid files from cur_dir_meta
            for n in missing:
                self.cur_dir_meta.remove(n)

        return entry_list

    def open_readable(self, name):
        """Open cur_dir/name for reading."""
        out = io.BytesIO()
        self.ftp.retrbinary("RETR %s" % name, out.write)
        out.flush()
        out.seek(0)
        return out

    def write_file(self,
                   name,
                   fp_src,
                   blocksize=DEFAULT_BLOCKSIZE,
                   callback=None):
        self.check_write(name)
        self.ftp.storbinary("STOR %s" % name, fp_src, blocksize, callback)
        # TODO: check result

    def remove_file(self, name):
        """Remove cur_dir/name."""
        self.check_write(name)
        #         self.cur_dir_meta.remove(name)
        self.ftp.delete(name)
        self.remove_sync_info(name)

    def set_mtime(self, name, mtime, size):
        self.check_write(name)
        #         print("META set_mtime(%s): %s" % (name, time.ctime(mtime)))
        # We cannot set the mtime on FTP servers, so we store this as additional
        # meta data in the same directory
        # TODO: try "SITE UTIME", "MDTM (set version)", or "SRFT" command
        self.cur_dir_meta.set_mtime(name, mtime, size)
Exemplo n.º 3
0
    def get_dir(self):
        entry_list = []
        entry_map = {}
        local_res = {
            "has_meta": False
        }  # pass local variables outside func scope

        def _addline(line):
            data, _, name = line.partition("; ")
            res_type = size = mtime = unique = None
            fields = data.split(";")
            # http://tools.ietf.org/html/rfc3659#page-23
            # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
            #   / "Media-Type" / "CharSet" / os-depend-fact / local-fact
            for field in fields:
                field_name, _, field_value = field.partition("=")
                field_name = field_name.lower()
                if field_name == "type":
                    res_type = field_value
                elif field_name in ("sizd", "size"):
                    size = int(field_value)
                elif field_name == "modify":
                    # Use calendar.timegm() instead of time.mktime(), because
                    # the date was returned as UTC
                    mtime = calendar.timegm(
                        time.strptime(field_value, "%Y%m%d%H%M%S"))
#                    print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
                elif field_name == "unique":
                    unique = field_value

            entry = None
            if res_type == "dir":
                entry = DirectoryEntry(self, self.cur_dir, name, size, mtime,
                                       unique)
            elif res_type == "file":
                if name == DirMetadata.META_FILE_NAME:
                    # the meta-data file is silently ignored
                    local_res["has_meta"] = True
                elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
                    entry = FileEntry(self, self.cur_dir, name, size, mtime,
                                      unique)
            elif res_type in ("cdir", "pdir"):
                pass
            else:
                raise NotImplementedError

            if entry:
                entry_map[name] = entry
                entry_list.append(entry)

        # raises error_perm, if command is not supported
        self.ftp.retrlines("MLSD", _addline)

        # load stored meta data if present
        self.cur_dir_meta = DirMetadata(self)

        if local_res["has_meta"]:
            try:
                self.cur_dir_meta.read()
            except Exception as e:
                print("Could not read meta info: %s" % e, file=sys.stderr)

            meta_files = self.cur_dir_meta.list

            # Adjust file mtime from meta-data if present
            missing = []
            for n in meta_files:
                meta = meta_files[n]
                if n in entry_map:
                    # We have a meta-data entry for this resource
                    upload_time = meta.get("u", 0)
                    # TODO: use 3 sec EPS, to compare mtimes
                    if entry_map[n].size == meta.get(
                            "s") and entry_map[n].mtime <= upload_time:
                        # Use meta-data mtime instead of the one reported by FTP server
                        entry_map[n].meta = meta
                        entry_map[n].mtime = meta["m"]
                    else:
                        # Discard stored meta-data if
                        #   1. the the mtime reported by the FTP server is later
                        #      than the stored upload time
                        #      or
                        #   2. the reported files size is different than the
                        #      size we stored in the meta-data
                        #                        print("META: Removing outdated meta entry %s" % n, meta)
                        missing.append(n)
                else:
                    # File is stored in meta-data, but no longer exists on FTP server
                    #                     print("META: Removing missing meta entry %s" % n)
                    missing.append(n)
            # Remove missing or invalid files from cur_dir_meta
            for n in missing:
                self.cur_dir_meta.remove(n)

        return entry_list
Exemplo n.º 4
0
class FtpTarget(_Target):
    """Represents a synchronisation target on a FTP server.

    Attributes:
        path (str): Current working directory on FTP server.
        ftp (FTP): Instance of ftplib.FTP.
        host (str): hostname of FTP server
        port (int): FTP port (defaults to 21)
        username (str):
        password (str):
    """
    def __init__(self, path, host, port=None, username=None, password=None,
                 tls=False, extra_opts=None):
        """Create FTP target with host, initial path, optional credentials and options.

        Args:
            path (str): root path on FTP server, relative to *host*
            host (str): hostname of FTP server
            port (int): FTP port (defaults to 21)
            username (str):
            password (str):
            tls (bool): encrypt the connection using TLS (Python 2.7/3.2+)
            extra_opts (dict):
        """
        path = path or "/"
        super(FtpTarget, self).__init__(path, extra_opts)
        if tls:
            try:
                self.ftp = ftplib.FTP_TLS()
            except AttributeError:
                print("Python 2.7/3.2+ required for FTPS (TLS).")
                raise
        else:
            self.ftp = ftplib.FTP()
        self.ftp.debug(self.get_option("ftp_debug", 0))
        self.host = host
        self.port = port
        self.username = username
        self.password = password
        self.tls = tls
        self.lock_data = None
        self.is_unix = None
        self.time_zone_ofs = None
        self.clock_ofs = None
#        if connect:
#            self.open()

    def __str__(self):
        return "<%s + %s>" % (self.get_base_name(),
                              relpath_url(self.cur_dir, self.root_dir))

    def get_base_name(self):
        scheme = "ftps" if self.tls else "ftp"
        return "%s://%s%s" % (scheme, self.host, self.root_dir)

    def open(self):
        assert not self.connected
        no_prompt  = self.get_option("no_prompt", True)
        store_password = self.get_option("store_password", False)

        if self.port:
            self.ftp.connect(self.host, int(self.port))
        else:
            self.ftp.connect(self.host)

        if self.username is None or self.password is None:
            creds = get_credentials_for_url(self.host, allow_prompt=not no_prompt)
            if creds:
                self.username, self.password = creds

        try:
            # Login (as 'anonymous' if self.username is undefined):
            self.ftp.login(self.username, self.password)
        except error_perm as e:
            # If credentials were passed, but authentication fails, prompt
            # for new password
            if not e.args[0].startswith("530"):
                raise # error other then '530 Login incorrect'
            print(e)
            if not no_prompt:
                self.user, self.password = prompt_for_password(self.host, self.username)
                self.ftp.login(self.username, self.password)

        if self.tls:
            # Upgrade data connection to TLS.
            self.ftp.prot_p()

        try:
            self.ftp.cwd(self.root_dir)
        except error_perm as e:
            # If credentials were passed, but authentication fails, prompt
            # for new password
            if not e.args[0].startswith("550"):
                raise # error other then 550 No such directory'
            print("Could not change directory to %s (%s): missing permissions?" % (self.root_dir, e))

        pwd = self.ftp.pwd()
        if pwd != self.root_dir:
            raise RuntimeError("Unable to navigate to working directory %r (now at %r)" % (self.root_dir, pwd))

        self.cur_dir = pwd
        self.connected = True
        # Successfully authenticated: store password
        if store_password:
            save_password(self.host, self.username, self.password)

        # TODO: case sensitivity?
#         resp = self.ftp.sendcmd("system")
#         self.is_unix = "unix" in resp.lower() # not necessarily true, better check with read/write tests

        self._lock()

        return

    def close(self):
        if self.connected:
            self._unlock()
            self.ftp.quit()
        self.connected = False

    def _lock(self, break_existing=False):
        """Write a special file to the target root folder.

        """
        data = {"lock_time": time.time(),
                "lock_holder": None}

        try:
            assert self.cur_dir == self.root_dir
            self.write_text(DirMetadata.LOCK_FILE_NAME, json.dumps(data))
            self.lock_data = data
        except Exception as e:
            print("Could not write lock file: %s" % e, file=sys.stderr)

    def _unlock(self):
        """Write a special file to the target root folder.

        """
        try:
            assert self.cur_dir == self.root_dir
            self.remove_file(DirMetadata.LOCK_FILE_NAME)
            self.lock_data = None
        except Exception as e:
            print("Could not remove lock file: %s" % e, file=sys.stderr)

    def _probe_lock_file(self, reported_mtime):
        """Called by get_dir"""
        delta = reported_mtime - self.lock_data["lock_time"]
        print("Server time offset: {0:.2f} seconds".format(delta))

    def get_id(self):
        return self.host + self.root_dir

    def cwd(self, dir_name):
        path = normpath_url(join_url(self.cur_dir, dir_name))
        if not path.startswith(self.root_dir):
            # paranoic check to prevent that our sync tool goes berserk
            raise RuntimeError("Tried to navigate outside root %r: %r"
                               % (self.root_dir, path))
        self.ftp.cwd(dir_name)
        self.cur_dir = path
        self.cur_dir_meta = None
        return self.cur_dir

    def pwd(self):
        return self.ftp.pwd()

    def mkdir(self, dir_name):
        self.check_write(dir_name)
        self.ftp.mkd(dir_name)

    def _rmdir_impl(self, dir_name, keep_root_folder=False, predicate=None):
        # FTP does not support deletion of non-empty directories.
#        print("rmdir(%s)" % dir_name)
        self.check_write(dir_name)
        names = self.ftp.nlst(dir_name)
#        print("rmdir(%s): %s" % (dir_name, names))
        # Skip ftp.cwd(), if dir is empty
        names = [ n for n in names if n not in (".", "..") ]
        if predicate:
            names = [ n for n in names if predicate(n) ]

        if len(names) > 0:
            self.ftp.cwd(dir_name)
            try:
                for name in names:
                    try:
                        # try to delete this as a file
                        self.ftp.delete(name)
                    except ftplib.all_errors as _e:
#                        print("    ftp.delete(%s) failed: %s, trying rmdir()..." % (name, _e))
                        # assume <name> is a folder
                        self.rmdir(name)
            finally:
                if dir_name != ".":
                    self.ftp.cwd("..")
#        print("ftp.rmd(%s)..." % (dir_name, ))
        if not keep_root_folder:
            self.ftp.rmd(dir_name)
        return


    def rmdir(self, dir_name):
        return self._rmdir_impl(dir_name)


    def get_dir(self):
        entry_list = []
        entry_map = {}
        local_res = {"has_meta": False} # pass local variables outside func scope

        def _addline(line):
            data, _, name = line.partition("; ")
            res_type = size = mtime = unique = None
            fields = data.split(";")
            # http://tools.ietf.org/html/rfc3659#page-23
            # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
            #   / "Media-Type" / "CharSet" / os-depend-fact / local-fact
            for field in fields:
                field_name, _, field_value = field.partition("=")
                field_name = field_name.lower()
                if field_name == "type":
                    res_type = field_value
                elif field_name in ("sizd", "size"):
                    size = int(field_value)
                elif field_name == "modify":
                    # Use calendar.timegm() instead of time.mktime(), because
                    # the date was returned as UTC
                    if "." in field_value:
                        mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
                    else:
                        mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S"))
#                    print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
                elif field_name == "unique":
                    unique = field_value

            entry = None
            if res_type == "dir":
                entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
            elif res_type == "file":
                if name == DirMetadata.META_FILE_NAME:
                    # the meta-data file is silently ignored
                    local_res["has_meta"] = True
                elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
                    # this is the root lock file. compare reported mtime with
                    # local upload time
                    self._probe_lock_file(mtime)
                elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
                    entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
            elif res_type in ("cdir", "pdir"):
                pass
            else:
                raise NotImplementedError

            if entry:
                entry_map[name] = entry
                entry_list.append(entry)

        # raises error_perm, if command is not supported
        self.ftp.retrlines("MLSD", _addline)

        # load stored meta data if present
        self.cur_dir_meta = DirMetadata(self)

        if local_res["has_meta"]:
            try:
                self.cur_dir_meta.read()
            except Exception as e:
                print("Could not read meta info: %s" % e, file=sys.stderr)

            meta_files = self.cur_dir_meta.list

            # Adjust file mtime from meta-data if present
            missing = []
            for n in meta_files:
                meta = meta_files[n]
                if n in entry_map:
                    # We have a meta-data entry for this resource
                    upload_time = meta.get("u", 0)
                    if(entry_map[n].size == meta.get("s") and
                           FileEntry._eps_compare(entry_map[n].mtime, upload_time) <= 0):
                        # Use meta-data mtime instead of the one reported by FTP server
                        entry_map[n].meta = meta
                        entry_map[n].mtime = meta["m"]
                    else:
                        # Discard stored meta-data if
                        #   1. the the mtime reported by the FTP server is later
                        #      than the stored upload time (which indicates
                        #      that the file was modified directly on the server)
                        #      or
                        #   2. the reported files size is different than the
                        #      size we stored in the meta-data
                        if self.synchronizer.verbose >= 3:
                            print(("META: Removing outdated meta entry %s\n" +
                                   "      modified after upload (%s > %s)") %
                                  (n, time.ctime(entry_map[n].mtime), time.ctime(upload_time)))
                        missing.append(n)
                else:
                    # File is stored in meta-data, but no longer exists on FTP server
#                     print("META: Removing missing meta entry %s" % n)
                    missing.append(n)
            # Remove missing or invalid files from cur_dir_meta
            for n in missing:
                self.cur_dir_meta.remove(n)

        return entry_list

    def open_readable(self, name):
        """Open cur_dir/name for reading."""
        out = io.BytesIO()
        self.ftp.retrbinary("RETR %s" % name, out.write)
        out.flush()
        out.seek(0)
        return out

    def write_file(self, name, fp_src, blocksize=DEFAULT_BLOCKSIZE, callback=None):
        self.check_write(name)
        self.ftp.storbinary("STOR %s" % name, fp_src, blocksize, callback)
        # TODO: check result

    def remove_file(self, name):
        """Remove cur_dir/name."""
        self.check_write(name)
#         self.cur_dir_meta.remove(name)
        self.ftp.delete(name)
        self.remove_sync_info(name)

    def set_mtime(self, name, mtime, size):
        self.check_write(name)
#         print("META set_mtime(%s): %s" % (name, time.ctime(mtime)))
        # We cannot set the mtime on FTP servers, so we store this as additional
        # meta data in the same directory
        # TODO: try "SITE UTIME", "MDTM (set version)", or "SRFT" command
        self.cur_dir_meta.set_mtime(name, mtime, size)
Exemplo n.º 5
0
    def get_dir(self):
        entry_list = []
        entry_map = {}
        local_res = {"has_meta": False} # pass local variables outside func scope

        def _addline(line):
            data, _, name = line.partition("; ")
            res_type = size = mtime = unique = None
            fields = data.split(";")
            # http://tools.ietf.org/html/rfc3659#page-23
            # "Size" / "Modify" / "Create" / "Type" / "Unique" / "Perm" / "Lang"
            #   / "Media-Type" / "CharSet" / os-depend-fact / local-fact
            for field in fields:
                field_name, _, field_value = field.partition("=")
                field_name = field_name.lower()
                if field_name == "type":
                    res_type = field_value
                elif field_name in ("sizd", "size"):
                    size = int(field_value)
                elif field_name == "modify":
                    # Use calendar.timegm() instead of time.mktime(), because
                    # the date was returned as UTC
                    if "." in field_value:
                        mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S.%f"))
                    else:
                        mtime = calendar.timegm(time.strptime(field_value, "%Y%m%d%H%M%S"))
#                    print("MLST modify: ", field_value, "mtime", mtime, "ctime", time.ctime(mtime))
                elif field_name == "unique":
                    unique = field_value

            entry = None
            if res_type == "dir":
                entry = DirectoryEntry(self, self.cur_dir, name, size, mtime, unique)
            elif res_type == "file":
                if name == DirMetadata.META_FILE_NAME:
                    # the meta-data file is silently ignored
                    local_res["has_meta"] = True
                elif name == DirMetadata.LOCK_FILE_NAME and self.cur_dir == self.root_dir:
                    # this is the root lock file. compare reported mtime with
                    # local upload time
                    self._probe_lock_file(mtime)
                elif not name in (DirMetadata.DEBUG_META_FILE_NAME, ):
                    entry = FileEntry(self, self.cur_dir, name, size, mtime, unique)
            elif res_type in ("cdir", "pdir"):
                pass
            else:
                raise NotImplementedError

            if entry:
                entry_map[name] = entry
                entry_list.append(entry)

        # raises error_perm, if command is not supported
        self.ftp.retrlines("MLSD", _addline)

        # load stored meta data if present
        self.cur_dir_meta = DirMetadata(self)

        if local_res["has_meta"]:
            try:
                self.cur_dir_meta.read()
            except Exception as e:
                print("Could not read meta info: %s" % e, file=sys.stderr)

            meta_files = self.cur_dir_meta.list

            # Adjust file mtime from meta-data if present
            missing = []
            for n in meta_files:
                meta = meta_files[n]
                if n in entry_map:
                    # We have a meta-data entry for this resource
                    upload_time = meta.get("u", 0)
                    if(entry_map[n].size == meta.get("s") and
                           FileEntry._eps_compare(entry_map[n].mtime, upload_time) <= 0):
                        # Use meta-data mtime instead of the one reported by FTP server
                        entry_map[n].meta = meta
                        entry_map[n].mtime = meta["m"]
                    else:
                        # Discard stored meta-data if
                        #   1. the the mtime reported by the FTP server is later
                        #      than the stored upload time (which indicates
                        #      that the file was modified directly on the server)
                        #      or
                        #   2. the reported files size is different than the
                        #      size we stored in the meta-data
                        if self.synchronizer.verbose >= 3:
                            print(("META: Removing outdated meta entry %s\n" +
                                   "      modified after upload (%s > %s)") %
                                  (n, time.ctime(entry_map[n].mtime), time.ctime(upload_time)))
                        missing.append(n)
                else:
                    # File is stored in meta-data, but no longer exists on FTP server
#                     print("META: Removing missing meta entry %s" % n)
                    missing.append(n)
            # Remove missing or invalid files from cur_dir_meta
            for n in missing:
                self.cur_dir_meta.remove(n)

        return entry_list