def test_parse_unix(self): parser = ListingParser(UNIX_LS) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) datetime_now = datetime.datetime.utcnow() datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc) current_year = datetime_now.year datetime_1 = date_factory(current_year, 1, 29, 3, 26) datetime_2 = date_factory(current_year, 1, 25, 0, 17) if datetime_1 > datetime_now: datetime_1 = datetime_1.replace(year=current_year - 1) if datetime_2 > datetime_now: datetime_2 = datetime_2.replace(year=current_year - 1) self.assertEqual([ FileEntry('README', 'file', 531, datetime_1, perm=0o644), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('bin', 'symlink', 7, datetime_2, 'usr/bin', perm=0o777), FileEntry('blah', 'dir', 512, date_factory(2004, 4, 8), perm=0o555), ], results)
def test_parse_nlst(self): parser = ListingParser(NLST) parser.run_heuristics() results = parser.parse() self.assertEqual([ FileEntry('dog.txt'), FileEntry('cat.txt'), FileEntry('bird.txt'), FileEntry('fish.txt'), ], results)
def test_parse_msdos_no_dir(self): parser = ListingParser(MSDOS_NO_DIR_LS) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual([ FileEntry('readme.htm', 'file', 589, date_factory(2000, 4, 14, 15, 47)), ], results)
def read_listing_content(self, file, duration_timeout=None): '''Read file listings. Returns: .ftp.request.ListingResponse: A Response populated the file listings Be sure to call :meth:`fetch_file_listing` first. Coroutine. ''' yield From( self.read_content(file=file, rewind=False, duration_timeout=duration_timeout)) try: if self._response.body.tell() == 0: listings = () elif self._listing_type == 'mlsd': self._response.body.seek(0) machine_listings = wpull.ftp.util.parse_machine_listing( self._response.body.read().decode( 'utf-8', errors='surrogateescape'), convert=True, strict=False) listings = list( wpull.ftp.util.machine_listings_to_file_entries( machine_listings)) else: self._response.body.seek(0) file = io.TextIOWrapper(self._response.body, encoding='utf-8', errors='surrogateescape') listing_parser = ListingParser(file=file) heuristics_result = listing_parser.run_heuristics() _logger.debug('Listing detected as %s', heuristics_result) listings = listing_parser.parse() # We don't want the file to be closed when exiting this function file.detach() except (ListingError, ValueError) as error: raise ProtocolError(*error.args) from error self._response.files = listings self._response.body.seek(0) raise Return(self._response)
def read_listing_content(self, file, duration_timeout=None): '''Read file listings. Returns: .ftp.request.ListingResponse: A Response populated the file listings Be sure to call :meth:`fetch_file_listing` first. Coroutine. ''' yield From(self.read_content(file=file, rewind=False, duration_timeout=duration_timeout)) try: if self._response.body.tell() == 0: listings = () elif self._listing_type == 'mlsd': self._response.body.seek(0) machine_listings = wpull.ftp.util.parse_machine_listing( self._response.body.read().decode('utf-8', errors='surrogateescape'), convert=True, strict=False ) listings = list( wpull.ftp.util.machine_listings_to_file_entries( machine_listings )) else: self._response.body.seek(0) file = io.TextIOWrapper(self._response.body, encoding='utf-8', errors='surrogateescape') listing_parser = ListingParser(file=file) heuristics_result = listing_parser.run_heuristics() _logger.debug('Listing detected as %s', heuristics_result) listings = listing_parser.parse() # We don't want the file to be closed when exiting this function file.detach() except (ListingError, ValueError) as error: raise ProtocolError(*error.args) from error self._response.files = listings self._response.body.seek(0) raise Return(self._response)
def test_parse_msdos_no_dir(self): parser = ListingParser(MSDOS_NO_DIR_LS) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual( [ FileEntry('readme.htm', 'file', 589, date_factory(2000, 4, 14, 15, 47)), ], results )
def test_parse_nlst(self): parser = ListingParser(NLST) parser.run_heuristics() results = parser.parse() self.assertEqual( [ FileEntry('dog.txt'), FileEntry('cat.txt'), FileEntry('bird.txt'), FileEntry('fish.txt'), ], results )
def test_parse_msdos(self): parser = ListingParser(MSDOS_LS) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual([ FileEntry('licensed', 'dir', None, date_factory( 2000, 4, 27, 21, 9)), FileEntry('pub', 'dir', None, date_factory(2000, 7, 18, 10, 16)), FileEntry('readme.htm', 'file', 589, date_factory(2000, 4, 14, 15, 47)), ], results)
def test_parse_msdos(self): parser = ListingParser(MSDOS_LS) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual( [ FileEntry('licensed', 'dir', None, date_factory(2000, 4, 27, 21, 9)), FileEntry('pub', 'dir', None, date_factory(2000, 7, 18, 10, 16)), FileEntry('readme.htm', 'file', 589, date_factory(2000, 4, 14, 15, 47)), ], results )
def test_parse_unix_datelike_file(self): parser = ListingParser(UNIX_LS_DATELIKE_FILE) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual( [ FileEntry('2009-12', 'file', 1558532, date_factory(2009, 12, 30), perm=0o644), FileEntry('2010-01', 'file', 10564020, date_factory(2010, 1, 14), perm=0o644), ], results )
def test_parse_unix(self): parser = ListingParser(UNIX_LS) parser.run_heuristics() results = parser.parse() date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) datetime_now = datetime.datetime.utcnow() datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc) current_year = datetime_now.year datetime_1 = date_factory(current_year, 1, 29, 3, 26) datetime_2 = date_factory(current_year, 1, 25, 0, 17) if datetime_1 > datetime_now: datetime_1 = datetime_1.replace(year=current_year - 1) if datetime_2 > datetime_now: datetime_2 = datetime_2.replace(year=current_year - 1) self.assertEqual( [ FileEntry('README', 'file', 531, datetime_1, perm=0o644), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('bin', 'symlink', 7, datetime_2, 'usr/bin', perm=0o777), FileEntry('blah', 'dir', 512, date_factory(2004, 4, 8), perm=0o555), ], results )
def test_parse_junk(self): parser = ListingParser(' aj \x00 a304 jrf') parser.run_heuristics() self.assertRaises(UnknownListingError, parser.parse)