def test_parse_unix(self):
parser = ListingParser(UNIX_LS)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
datetime_now = datetime.datetime.utcnow()
datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc)
current_year = datetime_now.year
datetime_1 = date_factory(current_year, 1, 29, 3, 26)
datetime_2 = date_factory(current_year, 1, 25, 0, 17)
if datetime_1 > datetime_now:
datetime_1 = datetime_1.replace(year=current_year - 1)
if datetime_2 > datetime_now:
datetime_2 = datetime_2.replace(year=current_year - 1)
self.assertEqual([
FileEntry('README', 'file', 531, datetime_1, perm=0o644),
FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555),
FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555),
FileEntry('bin', 'symlink', 7, datetime_2, 'usr/bin', perm=0o777),
FileEntry('blah', 'dir', 512, date_factory(2004, 4, 8),
perm=0o555),
], results)
def download_listing(self, file: Optional[IO], duration_timeout: Optional[float] = None) -> ListingResponse:
"""Read file listings.
Args:
file: A file object or asyncio stream.
duration_timeout: Maximum time in seconds of which the
entire file must be read.
Returns:
A Response populated the file listings
Be sure to call :meth:`start_file_listing` first.
Coroutine.
"""
if self._session_state != SessionState.directory_request_sent:
raise RuntimeError("File request not sent")
self._session_state = SessionState.file_request_sent
yield from self.download(file=file, rewind=False, duration_timeout=duration_timeout)
try:
if self._response.body.tell() == 0:
listings = ()
elif self._listing_type == "mlsd":
self._response.body.seek(0)
machine_listings = wpull.protocol.ftp.util.parse_machine_listing(
self._response.body.read().decode("utf-8", errors="surrogateescape"), convert=True, strict=False
)
listings = list(wpull.protocol.ftp.util.machine_listings_to_file_entries(machine_listings))
else:
self._response.body.seek(0)
file = io.TextIOWrapper(self._response.body, encoding="utf-8", errors="surrogateescape")
listing_parser = ListingParser(file=file)
listings = list(listing_parser.parse_input())
_logger.debug("Listing detected as %s", listing_parser.type)
# We don't want the file to be closed when exiting this function
file.detach()
except (ListingError, ValueError) as error:
raise ProtocolError(*error.args) from error
self._response.files = listings
self._response.body.seek(0)
self._session_state = SessionState.response_received
return self._response
def test_parse_nlst(self):
parser = ListingParser(NLST_LS)
results = list(parser.parse_input())
self.assertEqual([
FileEntry('horse.txt'),
FileEntry('fish'),
FileEntry('dolphin.jpg'),
FileEntry('delicious cake.wri'),
FileEntry('egg'),
], results)
def test_parse_msdos(self):
parser = ListingParser(MSDOS_LS)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual([
FileEntry('licensed', 'dir', None, date_factory(
2000, 4, 27, 21, 9)),
FileEntry('pub', 'dir', None, date_factory(2000, 7, 18, 10, 16)),
FileEntry('readme.htm', 'file', 589,
date_factory(2000, 4, 14, 15, 47)),
], results)
def test_parse_nlst(self):
parser = ListingParser(NLST_LS)
results = list(parser.parse_input())
self.assertEqual(
[
FileEntry('horse.txt'),
FileEntry('fish'),
FileEntry('dolphin.jpg'),
FileEntry('delicious cake.wri'),
FileEntry('egg'),
],
results
)
def test_parse_unix_datelike_file_2(self):
parser = ListingParser(UNIX_LS_DATELIKE_FILE_2)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual([
FileEntry('english_german.2010-03-24.tar.gz',
'file',
242408,
date_factory(2010, 3, 24),
perm=0o644),
FileEntry(
'old', 'dir', 4096, date_factory(2010, 3, 24), perm=0o755),
], results)
def test_parse_unix_datelike_file_2(self):
parser = ListingParser(UNIX_LS_DATELIKE_FILE_2)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('english_german.2010-03-24.tar.gz', 'file', 242408,
date_factory(2010, 3, 24),
perm=0o644),
FileEntry('old', 'dir', 4096,
date_factory(2010, 3, 24),
perm=0o755),
],
results
)
def test_parse_unix_datelike_file(self):
parser = ListingParser(UNIX_LS_DATELIKE_FILE)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('2009-12', 'file', 1558532,
date_factory(2009, 12, 30),
perm=0o644),
FileEntry('2010-01', 'file', 10564020,
date_factory(2010, 1, 14),
perm=0o644),
],
results
)
def test_parse_msdos_no_dir(self):
parser = ListingParser(MSDOS_NO_DIR_LS)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('licensed.exe', 'file', 123,
date_factory(2000, 4, 27, 21, 9)),
FileEntry('pub.pdf', 'file', 456,
date_factory(2000, 7, 18, 10, 16)),
FileEntry('readme.htm', 'file', 589,
date_factory(2000, 4, 14, 15, 47)),
],
results
)
def test_parse_unix_datelike_file(self):
parser = ListingParser(UNIX_LS_DATELIKE_FILE)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual([
FileEntry('2009-12',
'file',
1558532,
date_factory(2009, 12, 30),
perm=0o644),
FileEntry('2010-01',
'file',
10564020,
date_factory(2010, 1, 14),
perm=0o644),
], results)
def test_parse_unix(self):
parser = ListingParser(UNIX_LS)
results = list(parser.parse_input())
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
datetime_now = datetime.datetime.utcnow()
datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc)
current_year = datetime_now.year
datetime_1 = date_factory(current_year, 1, 29, 3, 26)
datetime_2 = date_factory(current_year, 1, 25, 0, 17)
if datetime_1 > datetime_now:
datetime_1 = datetime_1.replace(year=current_year - 1)
if datetime_2 > datetime_now:
datetime_2 = datetime_2.replace(year=current_year - 1)
self.assertEqual(
[
FileEntry('README', 'file', 531,
datetime_1,
perm=0o644),
FileEntry('etc', 'dir', 512,
date_factory(1994, 4, 8),
perm=0o555),
FileEntry('etc', 'dir', 512,
date_factory(1994, 4, 8),
perm=0o555),
FileEntry('bin', 'symlink', 7,
datetime_2,
'usr/bin', perm=0o777),
FileEntry('blah', 'dir', 512,
date_factory(2004, 4, 8),
perm=0o555),
],
results
)
def download_listing(self, file: Optional[IO],
duration_timeout: Optional[float]=None) -> \
ListingResponse:
'''Read file listings.
Args:
file: A file object or asyncio stream.
duration_timeout: Maximum time in seconds of which the
entire file must be read.
Returns:
A Response populated the file listings
Be sure to call :meth:`start_file_listing` first.
Coroutine.
'''
if self._session_state != SessionState.directory_request_sent:
raise RuntimeError('File request not sent')
self._session_state = SessionState.file_request_sent
yield from self.download(file=file,
rewind=False,
duration_timeout=duration_timeout)
try:
if self._response.body.tell() == 0:
listings = ()
elif self._listing_type == 'mlsd':
self._response.body.seek(0)
machine_listings = wpull.protocol.ftp.util.parse_machine_listing(
self._response.body.read().decode(
'utf-8', errors='surrogateescape'),
convert=True,
strict=False)
listings = list(
wpull.protocol.ftp.util.machine_listings_to_file_entries(
machine_listings))
else:
self._response.body.seek(0)
file = io.TextIOWrapper(self._response.body,
encoding='utf-8',
errors='surrogateescape')
listing_parser = ListingParser(file=file)
listings = list(listing_parser.parse_input())
_logger.debug('Listing detected as %s', listing_parser.type)
# We don't want the file to be closed when exiting this function
file.detach()
except (ListingError, ValueError) as error:
raise ProtocolError(*error.args) from error
self._response.files = listings
self._response.body.seek(0)
self._session_state = SessionState.response_received
return self._response