def test_parse_unix(self):
parser = ListingParser(UNIX_LS)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
datetime_now = datetime.datetime.utcnow()
datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc)
current_year = datetime_now.year
datetime_1 = date_factory(current_year, 1, 29, 3, 26)
datetime_2 = date_factory(current_year, 1, 25, 0, 17)
if datetime_1 > datetime_now:
datetime_1 = datetime_1.replace(year=current_year - 1)
if datetime_2 > datetime_now:
datetime_2 = datetime_2.replace(year=current_year - 1)
self.assertEqual([
FileEntry('README', 'file', 531, datetime_1, perm=0o644),
FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555),
FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555),
FileEntry('bin', 'symlink', 7, datetime_2, 'usr/bin', perm=0o777),
FileEntry('blah', 'dir', 512, date_factory(2004, 4, 8),
perm=0o555),
], results)
def test_parse_nlst(self):
parser = ListingParser(NLST)
parser.run_heuristics()
results = parser.parse()
self.assertEqual([
FileEntry('dog.txt'),
FileEntry('cat.txt'),
FileEntry('bird.txt'),
FileEntry('fish.txt'),
], results)
def test_parse_msdos_no_dir(self):
parser = ListingParser(MSDOS_NO_DIR_LS)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual([
FileEntry('readme.htm', 'file', 589,
date_factory(2000, 4, 14, 15, 47)),
], results)
def read_listing_content(self, file, duration_timeout=None):
'''Read file listings.
Returns:
.ftp.request.ListingResponse: A Response populated the
file listings
Be sure to call :meth:`fetch_file_listing` first.
Coroutine.
'''
yield From(
self.read_content(file=file,
rewind=False,
duration_timeout=duration_timeout))
try:
if self._response.body.tell() == 0:
listings = ()
elif self._listing_type == 'mlsd':
self._response.body.seek(0)
machine_listings = wpull.ftp.util.parse_machine_listing(
self._response.body.read().decode(
'utf-8', errors='surrogateescape'),
convert=True,
strict=False)
listings = list(
wpull.ftp.util.machine_listings_to_file_entries(
machine_listings))
else:
self._response.body.seek(0)
file = io.TextIOWrapper(self._response.body,
encoding='utf-8',
errors='surrogateescape')
listing_parser = ListingParser(file=file)
heuristics_result = listing_parser.run_heuristics()
_logger.debug('Listing detected as %s', heuristics_result)
listings = listing_parser.parse()
# We don't want the file to be closed when exiting this function
file.detach()
except (ListingError, ValueError) as error:
raise ProtocolError(*error.args) from error
self._response.files = listings
self._response.body.seek(0)
raise Return(self._response)
def read_listing_content(self, file, duration_timeout=None):
'''Read file listings.
Returns:
.ftp.request.ListingResponse: A Response populated the
file listings
Be sure to call :meth:`fetch_file_listing` first.
Coroutine.
'''
yield From(self.read_content(file=file, rewind=False,
duration_timeout=duration_timeout))
try:
if self._response.body.tell() == 0:
listings = ()
elif self._listing_type == 'mlsd':
self._response.body.seek(0)
machine_listings = wpull.ftp.util.parse_machine_listing(
self._response.body.read().decode('utf-8',
errors='surrogateescape'),
convert=True, strict=False
)
listings = list(
wpull.ftp.util.machine_listings_to_file_entries(
machine_listings
))
else:
self._response.body.seek(0)
file = io.TextIOWrapper(self._response.body, encoding='utf-8',
errors='surrogateescape')
listing_parser = ListingParser(file=file)
heuristics_result = listing_parser.run_heuristics()
_logger.debug('Listing detected as %s', heuristics_result)
listings = listing_parser.parse()
# We don't want the file to be closed when exiting this function
file.detach()
except (ListingError, ValueError) as error:
raise ProtocolError(*error.args) from error
self._response.files = listings
self._response.body.seek(0)
raise Return(self._response)
def test_parse_msdos_no_dir(self):
parser = ListingParser(MSDOS_NO_DIR_LS)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('readme.htm', 'file', 589,
date_factory(2000, 4, 14, 15, 47)),
],
results
)
def test_parse_nlst(self):
parser = ListingParser(NLST)
parser.run_heuristics()
results = parser.parse()
self.assertEqual(
[
FileEntry('dog.txt'),
FileEntry('cat.txt'),
FileEntry('bird.txt'),
FileEntry('fish.txt'),
],
results
)
def test_parse_msdos(self):
parser = ListingParser(MSDOS_LS)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual([
FileEntry('licensed', 'dir', None, date_factory(
2000, 4, 27, 21, 9)),
FileEntry('pub', 'dir', None, date_factory(2000, 7, 18, 10, 16)),
FileEntry('readme.htm', 'file', 589,
date_factory(2000, 4, 14, 15, 47)),
], results)
def test_parse_msdos(self):
parser = ListingParser(MSDOS_LS)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('licensed', 'dir', None,
date_factory(2000, 4, 27, 21, 9)),
FileEntry('pub', 'dir', None,
date_factory(2000, 7, 18, 10, 16)),
FileEntry('readme.htm', 'file', 589,
date_factory(2000, 4, 14, 15, 47)),
],
results
)
def test_parse_unix_datelike_file(self):
parser = ListingParser(UNIX_LS_DATELIKE_FILE)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('2009-12', 'file', 1558532,
date_factory(2009, 12, 30),
perm=0o644),
FileEntry('2010-01', 'file', 10564020,
date_factory(2010, 1, 14),
perm=0o644),
],
results
)
def test_parse_unix_datelike_file(self):
parser = ListingParser(UNIX_LS_DATELIKE_FILE)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
self.assertEqual(
[
FileEntry('2009-12', 'file', 1558532,
date_factory(2009, 12, 30),
perm=0o644),
FileEntry('2010-01', 'file', 10564020,
date_factory(2010, 1, 14),
perm=0o644),
],
results
)
def test_parse_unix(self):
parser = ListingParser(UNIX_LS)
parser.run_heuristics()
results = parser.parse()
date_factory = functools.partial(datetime.datetime,
tzinfo=datetime.timezone.utc)
datetime_now = datetime.datetime.utcnow()
datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc)
current_year = datetime_now.year
datetime_1 = date_factory(current_year, 1, 29, 3, 26)
datetime_2 = date_factory(current_year, 1, 25, 0, 17)
if datetime_1 > datetime_now:
datetime_1 = datetime_1.replace(year=current_year - 1)
if datetime_2 > datetime_now:
datetime_2 = datetime_2.replace(year=current_year - 1)
self.assertEqual(
[
FileEntry('README', 'file', 531,
datetime_1,
perm=0o644),
FileEntry('etc', 'dir', 512,
date_factory(1994, 4, 8),
perm=0o555),
FileEntry('etc', 'dir', 512,
date_factory(1994, 4, 8),
perm=0o555),
FileEntry('bin', 'symlink', 7,
datetime_2,
'usr/bin', perm=0o777),
FileEntry('blah', 'dir', 512,
date_factory(2004, 4, 8),
perm=0o555),
],
results
)
def test_parse_junk(self):
parser = ListingParser(' aj \x00 a304 jrf')
parser.run_heuristics()
self.assertRaises(UnknownListingError, parser.parse)
def test_parse_junk(self):
parser = ListingParser(' aj \x00 a304 jrf')
parser.run_heuristics()
self.assertRaises(UnknownListingError, parser.parse)
