def setUp(self): # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) FilePath(userdir).child('file.txt').setContent("I have the power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) p.registerChecker(users_checker, credentials.IUsernamePassword) self.factory = FTPFactory(portal=p) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening)
def setUp(self): # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) FilePath(userdir).child('file.txt').setContent("I have the power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) p.registerChecker(users_checker, credentials.IUsernamePassword) self.factory = FTPFactory(portal=p) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening)
def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dir and test file self.directory = self.mktemp() os.mkdir(self.directory) fp = FilePath(self.directory) fp.child('file.txt').setContent(b"I have the power!") fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server for anonymous access realm = FTPRealm(anonymousRoot=self.directory) p = portal.Portal(realm) p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous) self.factory = FTPFactory(portal=p, userAnonymous=self.username) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening)
class FTPTestCase(unittest.TestCase): username = "******" password = "******" if twisted_version < (10, 2, 0): skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" def setUp(self): # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) FilePath(userdir).child('file.txt').setContent("I have the power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) p.registerChecker(users_checker, credentials.IUsernamePassword) self.factory = FTPFactory(portal=p) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening) def _add_test_callbacks(self, deferred, callback=None, errback=None): def _clean(data): self.download_handler.client.transport.loseConnection() return data deferred.addCallback(_clean) if callback: deferred.addCallback(callback) if errback: deferred.addErrback(errback) return deferred def test_ftp_download_success(self): request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={ "ftp_user": self.username, "ftp_password": self.password }) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, 'I have the power!') self.assertEqual(r.headers, {'Local Filename': [''], 'Size': [17]}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, meta={ "ftp_user": self.username, "ftp_password": self.password }) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 404) return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): local_fname = "/tmp/file.txt" request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={ "ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname }) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.body, local_fname) self.assertEqual(r.headers, { 'Local Filename': ['/tmp/file.txt'], 'Size': [17] }) self.assertTrue(os.path.exists(local_fname)) with open(local_fname) as f: self.assertEqual(f.read(), "I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test) def test_invalid_credentials(self): request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={ "ftp_user": self.username, "ftp_password": '******' }) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.type, ConnectionLost) return self._add_test_callbacks(d, errback=_test)
class FTPTestCase(unittest.TestCase): username = "******" password = "******" if twisted_version < (10, 2, 0): skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home" def setUp(self): # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) fp.child("file.txt").setContent("I have the power!") fp.child("file with spaces.txt").setContent("Moooooooooo power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) p.registerChecker(users_checker, credentials.IUsernamePassword) self.factory = FTPFactory(portal=p) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening) def _add_test_callbacks(self, deferred, callback=None, errback=None): def _clean(data): self.download_handler.client.transport.loseConnection() return data deferred.addCallback(_clean) if callback: deferred.addCallback(callback) if errback: deferred.addErrback(errback) return deferred def test_ftp_download_success(self): request = Request( url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": self.password}, ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, "I have the power!") self.assertEqual(r.headers, {"Local Filename": [""], "Size": ["17"]}) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): request = Request( url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": self.password}, ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, "Moooooooooo power!") self.assertEqual(r.headers, {"Local Filename": [""], "Size": ["18"]}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): request = Request( url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": self.password}, ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 404) return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): local_fname = "/tmp/file.txt" request = Request( url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname}, ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.body, local_fname) self.assertEqual(r.headers, {"Local Filename": ["/tmp/file.txt"], "Size": ["17"]}) self.assertTrue(os.path.exists(local_fname)) with open(local_fname) as f: self.assertEqual(f.read(), "I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test) def test_invalid_credentials(self): request = Request( url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta={"ftp_user": self.username, "ftp_password": "******"}, ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.type, ConnectionLost) return self._add_test_callbacks(d, errback=_test)
class BaseFTPTestCase(unittest.TestCase): username = "******" password = "******" req_meta = {"ftp_user": username, "ftp_password": password} def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) fp.child('file.txt').setContent(b"I have the power!") fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) p.registerChecker(users_checker, credentials.IUsernamePassword) self.factory = FTPFactory(portal=p) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening) def tearDown(self): shutil.rmtree(self.directory) def _add_test_callbacks(self, deferred, callback=None, errback=None): def _clean(data): self.download_handler.client.transport.loseConnection() return data deferred.addCallback(_clean) if callback: deferred.addCallback(callback) if errback: deferred.addErrback(errback) return deferred def test_ftp_download_success(self): request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, { b'Local Filename': [b''], b'Size': [b'17'] }) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): request = Request(url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'Moooooooooo power!') self.assertEqual(r.headers, { b'Local Filename': [b''], b'Size': [b'18'] }) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 404) return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): f, local_fname = tempfile.mkstemp() local_fname = to_bytes(local_fname) os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta=meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.body, local_fname) self.assertEqual(r.headers, { b'Local Filename': [local_fname], b'Size': [b'17'] }) self.assertTrue(os.path.exists(local_fname)) with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test)
class BaseFTPTestCase(unittest.TestCase): username = "******" password = "******" req_meta = {"ftp_user": username, "ftp_password": password} def setUp(self): from twisted.protocols.ftp import FTPRealm, FTPFactory from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler # setup dirs and test file self.directory = self.mktemp() os.mkdir(self.directory) userdir = os.path.join(self.directory, self.username) os.mkdir(userdir) fp = FilePath(userdir) fp.child('file.txt').setContent(b"I have the power!") fp.child('file with spaces.txt').setContent(b"Moooooooooo power!") # setup server realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory) p = portal.Portal(realm) users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse() users_checker.addUser(self.username, self.password) p.registerChecker(users_checker, credentials.IUsernamePassword) self.factory = FTPFactory(portal=p) self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1") self.portNum = self.port.getHost().port self.download_handler = FTPDownloadHandler(Settings()) self.addCleanup(self.port.stopListening) def tearDown(self): shutil.rmtree(self.directory) def _add_test_callbacks(self, deferred, callback=None, errback=None): def _clean(data): self.download_handler.client.transport.loseConnection() return data deferred.addCallback(_clean) if callback: deferred.addCallback(callback) if errback: deferred.addErrback(errback) return deferred def test_ftp_download_success(self): request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'I have the power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']}) return self._add_test_callbacks(d, _test) def test_ftp_download_path_with_spaces(self): request = Request( url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum, meta=self.req_meta ) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 200) self.assertEqual(r.body, b'Moooooooooo power!') self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'18']}) return self._add_test_callbacks(d, _test) def test_ftp_download_notexist(self): request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum, meta=self.req_meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.status, 404) return self._add_test_callbacks(d, _test) def test_ftp_local_filename(self): f, local_fname = tempfile.mkstemp() local_fname = to_bytes(local_fname) os.close(f) meta = {"ftp_local_filename": local_fname} meta.update(self.req_meta) request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum, meta=meta) d = self.download_handler.download_request(request, None) def _test(r): self.assertEqual(r.body, local_fname) self.assertEqual(r.headers, {b'Local Filename': [local_fname], b'Size': [b'17']}) self.assertTrue(os.path.exists(local_fname)) with open(local_fname, "rb") as f: self.assertEqual(f.read(), b"I have the power!") os.remove(local_fname) return self._add_test_callbacks(d, _test)