def test_poke(self): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") self.hook_mock.get_mod_time.side_effect = \ [error_perm("550: Can't check for file existence"), error_perm("550: Directory or file does not exist"), error_perm("550 - Directory or file does not exist"), None] self.assertFalse(op.poke(None)) self.assertFalse(op.poke(None)) self.assertFalse(op.poke(None)) self.assertTrue(op.poke(None))
def test_poke(self, mock_hook): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") mock_hook.return_value.__enter__.return_value.get_mod_time.side_effect = [ error_perm("550: Can't check for file existence"), error_perm("550: Directory or file does not exist"), error_perm("550 - Directory or file does not exist"), None ] self.assertFalse(op.poke(None)) self.assertFalse(op.poke(None)) self.assertFalse(op.poke(None)) self.assertTrue(op.poke(None))
def storbinary(self, command, fp): if not self._logged_in: raise ftplib.error_perm('530 Please log in with USER and PASS first.') parts = command.split(' ', 1) if len(parts) == 2 and parts[0] == 'STOR': dst_path = _get_abs_url_path(self._url, parts[1]) try: content = fp.read() fsx.open(dst_path, 'wb').write(content) except EnvironmentError as exc: msg = '{exc.__class__.__name__}: {exc}'.format(exc=exc) raise ftplib.error_perm(msg) else: raise NotImplementedError("Commands other than 'STOR' are currently not implemented.")
def retrbinary(self, argument, callback): if not self._logged_in: raise ftplib.error_perm('530 Please log in with USER and PASS first.') parts = argument.split(' ', 1) if len(parts) == 2 and parts[0] == 'RETR': path = _get_abs_url_path(self._url, parts[1]) try: with fsx.open(path, 'rb') as file_: callback(file_.read()) except EnvironmentError as exc: msg = '{exc.__class__.__name__}: {exc}'.format(exc=exc) raise ftplib.error_perm(msg) else: raise NotImplementedError("Commands other than 'RETR' are currently not implemented.")
def test_no_new_files(self): try: self.ftp.retrbinary.side_effect = ftplib.error_perm("550 Failed to open file.") self.reader.get_frontfile_new( datetime.date(2113,11,4) ) self.fail("Exception expected") except ValueError, e: self.assertEqual("No new files entry was found for [2113-11-04]", e.message)
def handle_missing_date(self, f, dir_str): self.ftp.cwd.side_effect = [None, ftplib.error_perm("550 Failed to change directory.")] try: f() self.fail("Exception expected") except ValueError, e: self.assertEqual("No data found for given date. Target folder: [{}]".format(dir_str), e.message)
def test__create_folder_perm_err(self, mock_log, mock_ftp): mock_ftp.ftp.mkd.side_effect = ftplib.error_perm() expected = "" output = FTPUploader._create_folder(mock_ftp, "test") self.assertEqual(expected, output) self.assertTrue(mock_ftp.ftp.mkd.called) self.assertTrue(mock_log.debug.called)
def test_scandir_mlst_not_supported(self, mocked_ftp_conn): client = ftp_client.FTPClient("ftp://localhost:9999/mydir") with client: client.conn.mlsd.side_effect = ftplib.error_perm("501 'MLST type;'") client.scandir() # assert we do a simple dir in the client client.conn.dir.assert_called()
def test_no_new_files(self): try: self.ftp.retrbinary.side_effect = ftplib.error_perm( "550 Failed to open file.") self.reader.get_frontfile_new(datetime.date(2113, 11, 4)) self.fail("Exception expected") except ValueError, e: self.assertEqual("No new files entry was found for [2113-11-04]", e.message)
def test_poke_ignore_transient_error(self): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task", fail_on_transient_errors=False) self.hook_mock.get_mod_time.side_effect = \ [error_perm("434: Host unavailable"), None] self.assertFalse(op.poke(None)) self.assertTrue(op.poke(None))
def test_poke(self): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") self.hook_mock.get_mod_time.side_effect = \ [error_perm("550: Can't check for file existence"), None] self.assertFalse(op.poke(None)) self.assertTrue(op.poke(None))
def createFtpConnect(server, user, passwd): try: # login to ftp server ftp = FTP(server) ftp.login(user, passwd) return ftp except: print(error_perm())
def _download_ftp_dir(ftp_con, ftp_src, dest): print('Downloading dir: "{}" to: {}'.format(ftp_src, dest)) if not prepare_ftp_and_os_indexes(ftp_con, ftp_src, dest): return False file_list = get_file_list(ftp_con) if file_list is None or file_list is False: return False ftp_src_dir_name = MyGlobals.get_dir_name(ftp_src) print('Dir {} Files:\n -- {}\n\n'.format(ftp_src_dir_name, '\n -- '.join(file_list))) cur_ind = 0 end_ind = len(file_list) download_process_ok = True while (download_process_ok and cur_ind < end_ind): MyGlobals.sleep_for_a_while(MyGlobals.sleepBetweenDownloads_Default) cur_target_name = file_list[cur_ind] # Testing if it's a directory - then recurse try: target_path = ftp_src + "/" + cur_target_name # Check if a directory - if so then ftp-cd to it: if MyGlobals.isVerbose: print("Checking if {} is a directory".format(target_path)) is_dir = ftp_check_is_dir(ftp_con, target_path) if is_dir is None: print( 'Failed checking if {} is a directory'.format(target_path)) return False elif is_dir: if not change_ftp_dir(ftp_con, target_path): return False else: raise ftplib.error_perm( "target_path={} is a file".format(target_path)) download_process_ok = download_process_ok and _download_ftp_dir( ftp_con, target_path, dest + '/' + cur_target_name) if download_process_ok and not change_ftp_dir(ftp_con, ftp_src): return False # If got flag: --remove_src then Attempt to remove the dir after downloading it successfully if download_process_ok and MyGlobals.isRemoveSrc: download_process_ok = download_process_ok and ftp_delete_dir( ftp_con, target_path) except ftplib.error_perm: # It's a file: download_process_ok = download_process_ok and download_ftp_file( ftp_con, cur_target_name, dest) cur_ind += 1 return download_process_ok
def test__auth_fail_login(self, mock_report, mock_log, mock_ftp): mock_ftp.ftp.login.side_effect = ftplib.error_perm() with self.failUnlessRaises(FTPFailure): FTPUploader._auth(mock_ftp, "test", "test", "test", "test") self.assertTrue(mock_ftp.ftp.connect.called) self.assertTrue(mock_report.called) self.assertTrue(mock_log.error.called) self.assertTrue(mock_ftp.ftp.login.called) self.assertTrue(mock_log.debug.called)
def test_poke_ignore_transient_error(self, mock_hook): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task", fail_on_transient_errors=False) mock_hook.return_value.__enter__.return_value.get_mod_time.side_effect = [ error_perm("434: Host unavailable"), None ] self.assertFalse(op.poke(None)) self.assertTrue(op.poke(None))
def __init__(self, host, user, passwd): try: if user: self.ftp = ftplib.FTP(host, user, passwd) else: self.ftp = ftplib.FTP(host) self._generate_list() self._dir_path = '/' except ftplib.error_perm: raise ftplib.error_perm('Cannot connect. Incorrect information.')
def voidcmd(self, cmd): if DEBUG: print cmd if cmd == 'STAT': return 'MockSession server awaiting your commands ;-)' elif cmd.startswith('TYPE '): return elif cmd.startswith('SITE CHMOD'): raise ftplib.error_perm("502 command not implemented") else: raise ftplib.error_perm
def voidcmd(self, cmd): if DEBUG: print(cmd) if cmd == "STAT": return "MockSession server awaiting your commands ;-)" elif cmd.startswith("TYPE "): return elif cmd.startswith("SITE CHMOD"): raise ftplib.error_perm("502 command not implemented") else: raise ftplib.error_perm
def nlst(self, argument): if not self._logged_in: raise ftplib.error_perm('530 Please log in with USER and PASS first.') res = [] path = _get_abs_url_path(self._url, argument) for entry in fsx.scandir.scandir(path): res.append(entry.name) return res
def is_correct(password): server = ftplib.FTP() print(f"Trying: ", password) try: server.connect(host, port, timeout=5) server.login(user, password) except ftplib.error_perm(): return False else: print(f"{Fore.GREEN}Found credentials: ", password, Fore.RESET) return True
def test_poke_fails_due_error(self): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") self.hook_mock.get_mod_time.side_effect = \ error_perm("530: Login authentication failed") with self.assertRaises(error_perm) as context: op.execute(None) self.assertTrue("530" in str(context.exception))
def handle_missing_date(self, f, dir_str): self.ftp.cwd.side_effect = [ None, ftplib.error_perm("550 Failed to change directory.") ] try: f() self.fail("Exception expected") except ValueError, e: self.assertEqual( "No data found for given date. Target folder: [{}]".format( dir_str), e.message)
def test_scandir_dir_folder(self, mocked_ftp_conn): fake_entry = "drwxrwxrwx 1 owner group 0 Feb 17 17:54 nested" client = ftp_client.FTPClient("ftp://localhost:9999/mydir") with client: client.conn.mlsd.side_effect = ftplib.error_perm("501 'MLST type;'") # mock ftplib.dir behaviour client.conn.dir = lambda url, parser: parser(fake_entry) entries = list(client.scandir()) assert entries[0].url == URL("ftp://localhost:9999/mydir/nested") assert entries[0].is_dir
def test_poke_fail_on_transient_error(self, mock_hook): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") mock_hook.return_value.__enter__.return_value\ .get_mod_time.side_effect = error_perm("434: Host unavailable") with self.assertRaises(error_perm) as context: op.execute(None) self.assertTrue("434" in str(context.exception))
def test_ftp_correct_exception(self, mock_ftp): """set_initial_state() should not raise an error in case of 503 or 230 responses from FTP.login(), but it should for other error codes. """ test_crawler = crawlers.FTPCrawler('ftp://', username="******", password="******", include='\.gz$') mock_ftp.side_effect = ftplib.error_perm("503") test_crawler.set_initial_state() mock_ftp.side_effect = ftplib.error_perm("230") test_crawler.set_initial_state() mock_ftp.side_effect = ftplib.error_perm("999") with self.assertRaises(ftplib.error_perm): test_crawler.set_initial_state()
def test_poke_fail_on_transient_error(self): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") self.hook_mock.get_mod_time.side_effect = \ error_perm("434: Host unavailable") with self.assertRaises(error_perm) as context: op.execute(None) self.assertTrue("434" in str(context.exception))
def test_poke_fails_due_error(self, mock_hook): op = FTPSensor(path="foobar.json", ftp_conn_id="bob_ftp", task_id="test_task") mock_hook.return_value.__enter__.return_value.get_mod_time.side_effect = error_perm( "530: Login authentication failed") with pytest.raises(error_perm) as ctx: op.execute(None) assert "530" in str(ctx.value)
def test_ftplib_error_to_ftp_os_error_non_ascii_server_message(self): """ Test that we don't get a `UnicodeDecodeError` if the server sends a message containing non-ASCII characters. """ # See ticket #77. message = \ ftputil.tool.as_bytes( "Não é possível criar um arquivo já existente.") with pytest.raises(ftputil.error.PermanentError): with ftputil.error.ftplib_error_to_ftp_os_error: raise ftplib.error_perm(message)
def test_get_mtime(ftp_client): ftp_client.sendcmd.return_value = '213 20180101203000' assert ftp_mtime(url='foo') == 1_514_838_600 ftp_client.sendcmd.return_value = '213 20180101203000.123' assert ftp_mtime(url='foo') == 1_514_838_600 ftp_client.sendcmd.side_effect = AttributeError ftp_client.stat.return_value.st_mtime = 1_514_835_000 assert ftp_mtime(url='foo') == 1_514_835_000 ftp_client.sendcmd.side_effect = ftplib.error_perm('zbruh') assert ftp_mtime(url='foo') is None
def test_manager(self): mem_fs = open_fs('mem://') with self.assertRaises(errors.ResourceError): with ftp_errors(mem_fs, path='foo'): raise error_temp with self.assertRaises(errors.OperationFailed): with ftp_errors(mem_fs): raise error_temp with self.assertRaises(errors.InsufficientStorage): with ftp_errors(mem_fs): raise error_perm('552 foo') with self.assertRaises(errors.ResourceNotFound): with ftp_errors(mem_fs): raise error_perm('501 foo') with self.assertRaises(errors.PermissionDenied): with ftp_errors(mem_fs): raise error_perm('999 foo')
def test_get_mtime(ftp_client): ftp_client.sendcmd.return_value = "213 20180101203000" assert ftp_mtime(url="foo") == 1_514_838_600 ftp_client.sendcmd.return_value = "213 20180101203000.123" assert ftp_mtime(url="foo") == 1_514_838_600 ftp_client.sendcmd.side_effect = AttributeError ftp_client.stat.return_value.st_mtime = 1_514_835_000 assert ftp_mtime(url="foo") == 1_514_835_000 ftp_client.sendcmd.side_effect = ftplib.error_perm("zbruh") assert ftp_mtime(url="foo") is None
def searchPubmed(searchFields, sortby='relevance', num="10", resultsFormat='json'): """ Searches PubMed database for MeSH terms and other additional fields ('searchFields'), sorts them by relevance and \ returns the top 'num'. :param list searchFields: list of search fields to query for. :param str sortby: parameter to use for sorting. :param str num: number of PubMed identifiers to return. :param str resultsFormat: format of the PubMed result. :return: Dictionary with total number of PubMed ids, and top 'num' ids. """ pubmedQueryUrl = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=TERM&retmode=json&retmax=NUM' if len(searchFields) > 1: query = " [MeSH Terms] AND ".join(searchFields) else: query = searchFields[0] + " [MeSH Terms] AND" try: url = pubmedQueryUrl.replace('TERMS', query).replace('NUM', num) http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) response = http.request("GET", urllib.parse.quote(url)) jsonResponse = response.read() resultDict = json.loads(jsonResponse) except urllib3.exceptions.InvalidHeader: raise urllib3.exceptions.InvalidHeader("Invalid HTTP header provided. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.ConnectTimeoutError: raise urllib3.exceptions.ConnectTimeoutError("Connection timeout requesting URL. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.ConnectionError: raise urllib3.exceptions.ConnectionError("Protocol error when downloading. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.DecodeError: raise urllib3.exceptions.DecodeError("Decoder error when downloading. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.SecurityWarning: raise urllib3.exceptions.SecurityWarning("Security warning when downloading. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.ProtocolError: raise urllib3.exceptions.ProtocolError("Protocol error when downloading. {}.\nURL:{}".format(err,url)) except ftplib.error_reply as err: raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,url)) except ftplib.error_temp as err: raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,url)) except ftplib.error_perm as err: raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,url)) except ftplib.error_proto: raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,url)) except Exception as err: raise Exception("Something went wrong. {}.\nURL:{}".format(err,url)) result = [] if 'esearchresult' in resultDict: result = resultDict['esearchresult'] return result
def download_from_ftp(ftp_url, user, password, to, file_name): try: domain = ftp_url.split('/')[2] ftp_file = '/'.join(ftp_url.split('/')[3:]) with ftplib.FTP(domain) as ftp: ftp.login(user=user, passwd=password) with open(os.path.join(to, file_name), 'wb') as fp: ftp.retrbinary("RETR " + ftp_file, fp.write) except ftplib.error_reply as err: raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,ftp_url)) except ftplib.error_temp as err: raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,ftp_url)) except ftplib.error_perm as err: raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,ftp_url)) except ftplib.error_proto: raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,ftp_url))
def getresp(self): resp = self.getmultiline() if self.debugging: print('*resp*', self.sanitize(resp)) self.lastresp = resp[:3] c = resp[:1] if c in {'1', '2', '3'}: return resp if c == '4': raise ftplib.error_temp(resp) if c == '5': if self.lastresp == '553' and self.use_xdupe: with self.lock: self.parse_xdupe(resp) raise ftplib.error_perm(resp) raise ftplib.error_proto(resp)
def listfile(self): """ See if filename is in the current FTP directory. """ if not self.filename: return files = self.get_files() log.debug(LOG_CHECK, "FTP files %s", str(files)) if self.filename in files: # file found return # it could be a directory if the trailing slash was forgotten if "%s/" % self.filename in files: if not self.url.endswith("/"): self.add_warning(_("Missing trailing directory slash in ftp url."), tag=WARN_FTP_MISSING_SLASH) self.url += "/" return raise ftplib.error_perm("550 File not found")
def listfile(self): """ See if filename is in the current FTP directory. """ if not self.filename: return files = self.get_files() log.debug(LOG_CHECK, "FTP files %s", str(files)) if self.filename in files: # file found return # it could be a directory if the trailing slash was forgotten if "%s/" % self.filename in files: if not self.url.endswith('/'): self.add_warning( _("Missing trailing directory slash in ftp url."), tag=WARN_FTP_MISSING_SLASH) self.url += '/' return raise ftplib.error_perm("550 File not found")
def getresp(self): """ ftplib.getresp : parse JOBNAME in 250/125 z/OS FTP response """ resp = self.getmultiline() if self.debugging: print('*resp*', self.sanitize(resp)) self.lastresp = resp[:3] c = resp[:1] if c in ('1', '2', '3'): if resp[:3] in('250','125'): #|Zftp spec sx0 = re.search(r"\s+(JOB\d{5})\s+", resp) #| if sx0: #| self.__jobid = sx0.group(1) #| return resp if c == '4': raise ftplib.error_temp(resp) if c == '5': raise ftplib.error_perm(resp) raise ftplib.error_proto(resp)
def test_move_files(self, mocked_ftplib): client = mocked_ftplib().__enter__() # simulate file is not already there client.nlst.side_effect = ftplib.error_perm() to_move = "move/from/path/myfile.txt" to_path = "move/to/path" self.backend.move_files([to_move], to_path) # no need to delete it client.delete.assert_not_called() # rename gets called client.rename.assert_called_with( "upload/" + to_move, "upload/" + to_move.replace("from", "to")) # now try to override destination client.nlst.side_effect = None client.nlst.return_value = True self.backend.move_files([to_move], to_path) # client will delete it first client.delete.assert_called_with(to_move.replace("from", "to")) # then move it client.rename.assert_called_with( "upload/" + to_move, "upload/" + to_move.replace("from", "to"))
def cd(self, path): try: self.ftp.cwd(path) if path[0] == '/': self._dir_path = path elif '/' in path and not '..' in path: self._dir_path = self._dir_path + '/' + path elif '..' in path and len(self._dir_path) > 1: temp = self._dir_path.split('/') temp.pop() self._dir_path = '/'.join(temp) if len(self._dir_path) == 0: self._dir_path = '/' if self._dir_path.index('/') != 0: self._dir_path.insert(0, '/') if len(path[2:]) > 0: if len(self._dir_path) > 0 and self._dir_path[len(self._dir_path) -1] != '/': self._dir_path = self._dir_path + '/' + path[3:] else: self._dir_path += path[3:] elif self._dir_path == '/': self._dir_path += path else: self._dir_path = self._dir_path + '/' + path self._generate_list() except ftplib.error_perm: raise ftplib.error_perm('The path does not exist.')
def _get_unicode_data_file(file_name, version=None, force=False): if version: url_dir = '/Public/' + str(version) + '/ucd/' else: url_dir = '/Public/UCD/latest/ucd/' path_list = ['', 'extracted/', 'auxiliary/'] ftp = ftplib.FTP('ftp.unicode.org') ftp.login() ftp.cwd(url_dir) file_found = False for path in path_list: ftp.cwd(url_dir + path) try: with open(data_directory + file_name, 'wb') as out_file: ftp.retrbinary("RETR " + file_name, out_file.write) file_found = True break except ftplib.error_perm: pass if not file_found: ftp.cwd(url_dir) if force: with open(data_directory + 'Unihan.zip', 'wb') as zip_file: ftp.retrbinary("RETR Unihan.zip", zip_file.write) elif os.path.exists(data_directory + 'Unihan.zip'): pass else: with open(data_directory + 'Unihan.zip', 'wb') as zip_file: ftp.retrbinary("RETR Unihan.zip", zip_file.write) with open(data_directory + file_name, 'wb') as out_file, zipfile.ZipFile(data_directory + 'Unihan.zip') as zip_file: try: data = zip_file.open(file_name, 'r').read() out_file.write(data) file_found = True except: pass if not file_found: raise ftplib.error_perm(file_name + ' not found in Unicode data repository')
def ftpRetreive(filename, ftp, outfile=None): try: ftp.retrlines('RETR ' + filename, open(filename, 'wb').write) print("Download complete!") except: print(error_perm())
def callee(self): raise ftplib.error_perm()
def rmdir(self, dirname): try: self.ftp.rmd(dirname) self._generate_list() except ftplib.error_perm: raise ftplib.error_perm('Directory does not exist.')
def rm(self, file): try: self.ftp.delete(file) self._generate_list() except ftplib.error_perm: raise ftplib.error_perm('File does not exist.')
def cwd(self, dir): # Assume that `dir` is the inaccessible login directory. raise ftplib.error_perm("can't change into this directory")
def ftpTransfer(filename, ftp): try: ftp.transfercmd('STOR ' + filename) print("Upload complete!") except: print(error_perm())