def test_parse_unix(self): parser = ListingParser(UNIX_LS) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) datetime_now = datetime.datetime.utcnow() datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc) current_year = datetime_now.year datetime_1 = date_factory(current_year, 1, 29, 3, 26) datetime_2 = date_factory(current_year, 1, 25, 0, 17) if datetime_1 > datetime_now: datetime_1 = datetime_1.replace(year=current_year - 1) if datetime_2 > datetime_now: datetime_2 = datetime_2.replace(year=current_year - 1) self.assertEqual([ FileEntry('README', 'file', 531, datetime_1, perm=0o644), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('bin', 'symlink', 7, datetime_2, 'usr/bin', perm=0o777), FileEntry('blah', 'dir', 512, date_factory(2004, 4, 8), perm=0o555), ], results)
def download_listing(self, file: Optional[IO], duration_timeout: Optional[float] = None) -> ListingResponse: """Read file listings. Args: file: A file object or asyncio stream. duration_timeout: Maximum time in seconds of which the entire file must be read. Returns: A Response populated the file listings Be sure to call :meth:`start_file_listing` first. Coroutine. """ if self._session_state != SessionState.directory_request_sent: raise RuntimeError("File request not sent") self._session_state = SessionState.file_request_sent yield from self.download(file=file, rewind=False, duration_timeout=duration_timeout) try: if self._response.body.tell() == 0: listings = () elif self._listing_type == "mlsd": self._response.body.seek(0) machine_listings = wpull.protocol.ftp.util.parse_machine_listing( self._response.body.read().decode("utf-8", errors="surrogateescape"), convert=True, strict=False ) listings = list(wpull.protocol.ftp.util.machine_listings_to_file_entries(machine_listings)) else: self._response.body.seek(0) file = io.TextIOWrapper(self._response.body, encoding="utf-8", errors="surrogateescape") listing_parser = ListingParser(file=file) listings = list(listing_parser.parse_input()) _logger.debug("Listing detected as %s", listing_parser.type) # We don't want the file to be closed when exiting this function file.detach() except (ListingError, ValueError) as error: raise ProtocolError(*error.args) from error self._response.files = listings self._response.body.seek(0) self._session_state = SessionState.response_received return self._response
def test_parse_nlst(self): parser = ListingParser(NLST_LS) results = list(parser.parse_input()) self.assertEqual([ FileEntry('horse.txt'), FileEntry('fish'), FileEntry('dolphin.jpg'), FileEntry('delicious cake.wri'), FileEntry('egg'), ], results)
def test_parse_msdos(self): parser = ListingParser(MSDOS_LS) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual([ FileEntry('licensed', 'dir', None, date_factory( 2000, 4, 27, 21, 9)), FileEntry('pub', 'dir', None, date_factory(2000, 7, 18, 10, 16)), FileEntry('readme.htm', 'file', 589, date_factory(2000, 4, 14, 15, 47)), ], results)
def test_parse_nlst(self): parser = ListingParser(NLST_LS) results = list(parser.parse_input()) self.assertEqual( [ FileEntry('horse.txt'), FileEntry('fish'), FileEntry('dolphin.jpg'), FileEntry('delicious cake.wri'), FileEntry('egg'), ], results )
def test_parse_unix_datelike_file_2(self): parser = ListingParser(UNIX_LS_DATELIKE_FILE_2) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual([ FileEntry('english_german.2010-03-24.tar.gz', 'file', 242408, date_factory(2010, 3, 24), perm=0o644), FileEntry( 'old', 'dir', 4096, date_factory(2010, 3, 24), perm=0o755), ], results)
def test_parse_unix_datelike_file_2(self): parser = ListingParser(UNIX_LS_DATELIKE_FILE_2) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual( [ FileEntry('english_german.2010-03-24.tar.gz', 'file', 242408, date_factory(2010, 3, 24), perm=0o644), FileEntry('old', 'dir', 4096, date_factory(2010, 3, 24), perm=0o755), ], results )
def test_parse_unix_datelike_file(self): parser = ListingParser(UNIX_LS_DATELIKE_FILE) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual( [ FileEntry('2009-12', 'file', 1558532, date_factory(2009, 12, 30), perm=0o644), FileEntry('2010-01', 'file', 10564020, date_factory(2010, 1, 14), perm=0o644), ], results )
def test_parse_msdos_no_dir(self): parser = ListingParser(MSDOS_NO_DIR_LS) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual( [ FileEntry('licensed.exe', 'file', 123, date_factory(2000, 4, 27, 21, 9)), FileEntry('pub.pdf', 'file', 456, date_factory(2000, 7, 18, 10, 16)), FileEntry('readme.htm', 'file', 589, date_factory(2000, 4, 14, 15, 47)), ], results )
def test_parse_unix_datelike_file(self): parser = ListingParser(UNIX_LS_DATELIKE_FILE) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) self.assertEqual([ FileEntry('2009-12', 'file', 1558532, date_factory(2009, 12, 30), perm=0o644), FileEntry('2010-01', 'file', 10564020, date_factory(2010, 1, 14), perm=0o644), ], results)
def test_parse_unix(self): parser = ListingParser(UNIX_LS) results = list(parser.parse_input()) date_factory = functools.partial(datetime.datetime, tzinfo=datetime.timezone.utc) datetime_now = datetime.datetime.utcnow() datetime_now = datetime_now.replace(tzinfo=datetime.timezone.utc) current_year = datetime_now.year datetime_1 = date_factory(current_year, 1, 29, 3, 26) datetime_2 = date_factory(current_year, 1, 25, 0, 17) if datetime_1 > datetime_now: datetime_1 = datetime_1.replace(year=current_year - 1) if datetime_2 > datetime_now: datetime_2 = datetime_2.replace(year=current_year - 1) self.assertEqual( [ FileEntry('README', 'file', 531, datetime_1, perm=0o644), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('etc', 'dir', 512, date_factory(1994, 4, 8), perm=0o555), FileEntry('bin', 'symlink', 7, datetime_2, 'usr/bin', perm=0o777), FileEntry('blah', 'dir', 512, date_factory(2004, 4, 8), perm=0o555), ], results )
def download_listing(self, file: Optional[IO], duration_timeout: Optional[float]=None) -> \ ListingResponse: '''Read file listings. Args: file: A file object or asyncio stream. duration_timeout: Maximum time in seconds of which the entire file must be read. Returns: A Response populated the file listings Be sure to call :meth:`start_file_listing` first. Coroutine. ''' if self._session_state != SessionState.directory_request_sent: raise RuntimeError('File request not sent') self._session_state = SessionState.file_request_sent yield from self.download(file=file, rewind=False, duration_timeout=duration_timeout) try: if self._response.body.tell() == 0: listings = () elif self._listing_type == 'mlsd': self._response.body.seek(0) machine_listings = wpull.protocol.ftp.util.parse_machine_listing( self._response.body.read().decode( 'utf-8', errors='surrogateescape'), convert=True, strict=False) listings = list( wpull.protocol.ftp.util.machine_listings_to_file_entries( machine_listings)) else: self._response.body.seek(0) file = io.TextIOWrapper(self._response.body, encoding='utf-8', errors='surrogateescape') listing_parser = ListingParser(file=file) listings = list(listing_parser.parse_input()) _logger.debug('Listing detected as %s', listing_parser.type) # We don't want the file to be closed when exiting this function file.detach() except (ListingError, ValueError) as error: raise ProtocolError(*error.args) from error self._response.files = listings self._response.body.seek(0) self._session_state = SessionState.response_received return self._response
def test_parse_junk(self): parser = ListingParser(' aj \x00 a304 jrf') self.assertRaises(UnknownListingError, parser.parse_input)