def _list_files(self, rsync, path):
"""
This method recursively retrieves a list of files contained in a
directory, either local or remote (if starts with ':')
:param Rsync rsync: the Rsync object used to retrieve the list
:param str path: the path we want to inspect
:except CommandFailedException: if rsync call fails
:except RsyncListFilesFailure: if rsync output can't be parsed
"""
_logger.debug("list_files: %r", path)
# Use the --no-human-readable option to avoid digit groupings
# in "size" field with rsync >= 3.1.0.
# Ref: http://ftp.samba.org/pub/rsync/src/rsync-3.1.0-NEWS
rsync.get_output('--no-human-readable',
'--list-only',
'-r',
path,
check=True)
# Cache tzlocal object we need to build dates
tzinfo = dateutil.tz.tzlocal()
for line in rsync.out.splitlines():
line = line.rstrip()
match = self.LIST_ONLY_RE.match(line)
if match:
mode = match.group('mode')
# no exceptions here: the regexp forces 'size' to be an integer
size = int(match.group('size'))
try:
date_str = match.group('date')
# The date format has been validated by LIST_ONLY_RE.
# Use "2014/06/05 18:00:00" format if the sending rsync
# is compiled with HAVE_STRFTIME, otherwise use
# "Thu Jun 5 18:00:00 2014" format
if date_str[0].isdigit():
date = datetime.datetime.strptime(
date_str, "%Y/%m/%d %H:%M:%S")
else:
date = datetime.datetime.strptime(
date_str, "%a %b %d %H:%M:%S %Y")
date = date.replace(tzinfo=tzinfo)
except (TypeError, ValueError):
# This should not happen, due to the regexp
msg = ("Unable to parse rsync --list-only output line "
"(date): '%s'" % line)
_logger.exception(msg)
raise RsyncListFilesFailure(msg)
path = match.group('path')
yield _FileItem(mode, size, date, path)
else:
# This is a hard error, as we are unable to parse the output
# of rsync. It can only happen with a modified or unknown
# rsync version (perhaps newer than 3.1?)
msg = ("Unable to parse rsync --list-only output line: "
"'%s'" % line)
_logger.error(msg)
raise RsyncListFilesFailure(msg)
def list_files(self, path):
"""
This method recursively retrieves a list of files contained in a
directory, either local or remote (if starts with ':')
:param str path: the path we want to inspect
:except CommandFailedException: if rsync call fails
:except RsyncListFilesFailure: if rsync output can't be parsed
"""
_logger.debug("list_files: %r", path)
# Use the --no-human-readable option to avoid digit groupings
# in "size" field with rsync >= 3.1.0.
# Ref: http://ftp.samba.org/pub/rsync/src/rsync-3.1.0-NEWS
self.getoutput('--no-human-readable', '--list-only', '-r', path,
check=True)
for line in self.out.splitlines():
line = line.rstrip()
match = self.LIST_ONLY_RE.match(line)
if match:
mode = match.group('mode')
# no exceptions here: the regexp forces 'size' to be an integer
size = int(match.group('size'))
try:
date = dateutil.parser.parse(match.group('date'))
date = date.replace(tzinfo=dateutil.tz.tzlocal())
except (TypeError, ValueError):
# This should not happen, due to the regexp
msg = ("Unable to parse rsync --list-only output line "
"(date): '%s'" % line)
_logger.exception(msg)
raise RsyncListFilesFailure(msg)
path = match.group('path')
yield self.FileItem(mode, size, date, path)
else:
# This is a hard error, as we are unable to parse the output
# of rsync. It can only happen with a modified or unknown
# rsync version (perhaps newer than 3.1?)
msg = ("Unable to parse rsync --list-only output line: "
"'%s'" % line)
_logger.error(msg)
raise RsyncListFilesFailure(msg)
def _list_files(self, item, path):
"""
This method recursively retrieves a list of files contained in a
directory, either local or remote (if starts with ':')
:param _RsyncCopyItem item: information about a copy operation
:param str path: the path we want to inspect
:except CommandFailedException: if rsync call fails
:except RsyncListFilesFailure: if rsync output can't be parsed
"""
_logger.debug("list_files: %r", path)
# Build the rsync object required for the analysis
rsync = self._rsync_factory(item)
try:
# Use the --no-human-readable option to avoid digit groupings
# in "size" field with rsync >= 3.1.0.
# Ref: http://ftp.samba.org/pub/rsync/src/rsync-3.1.0-NEWS
rsync.get_output(
"--no-human-readable", "--list-only", "-r", path, check=True
)
except CommandFailedException:
# This could fail due to the local or the remote rsync
# older than 3.1. IF so, fallback to pre 3.1 mode
if self.rsync_has_ignore_missing_args and rsync.ret in (
12, # Error in rsync protocol data stream (remote)
1,
): # Syntax or usage error (local)
self._rsync_set_pre_31_mode()
# Recursive call, uses the compatibility mode
for item in self._list_files(item, path):
yield item
return
else:
raise
# Cache tzlocal object we need to build dates
tzinfo = dateutil.tz.tzlocal()
for line in rsync.out.splitlines():
line = line.rstrip()
match = self.LIST_ONLY_RE.match(line)
if match:
mode = match.group("mode")
# no exceptions here: the regexp forces 'size' to be an integer
size = int(match.group("size"))
try:
date_str = match.group("date")
# The date format has been validated by LIST_ONLY_RE.
# Use "2014/06/05 18:00:00" format if the sending rsync
# is compiled with HAVE_STRFTIME, otherwise use
# "Thu Jun 5 18:00:00 2014" format
if date_str[0].isdigit():
date = datetime.datetime.strptime(date_str, "%Y/%m/%d %H:%M:%S")
else:
date = datetime.datetime.strptime(
date_str, "%a %b %d %H:%M:%S %Y"
)
date = date.replace(tzinfo=tzinfo)
except (TypeError, ValueError):
# This should not happen, due to the regexp
msg = (
"Unable to parse rsync --list-only output line "
"(date): '%s'" % line
)
_logger.exception(msg)
raise RsyncListFilesFailure(msg)
path = match.group("path")
yield _FileItem(mode, size, date, path)
else:
# This is a hard error, as we are unable to parse the output
# of rsync. It can only happen with a modified or unknown
# rsync version (perhaps newer than 3.1?)
msg = "Unable to parse rsync --list-only output line: '%s'" % line
_logger.error(msg)
raise RsyncListFilesFailure(msg)