def __wrap_reconnect(self, method: Callable[[ftplib.FTP], T]) -> T: """ Dispatche the method to the connection and reconnects if needed. :param method: to be dispatched :return: response from the method """ last_err = None # type: Optional[Exception] for _ in range(0, self.max_reconnects): try: if self.connection is None: self.connect() assert self.connection is not None, "Expected connect() to either raise or create a connection" return method(self.connection) except (ConnectionRefusedError, ConnectionAbortedError, socket.timeout, socket.gaierror, socket.herror, ftplib.error_temp, EOFError) as err: self.connection.close() self.connection = None last_err = err assert last_err is not None, 'Expected either an error or a previous return' raise ftplib.error_temp( "Failed to execute a command on {}:{} after {} reconnect(s), the last error was: {}" .format(self.access.hostname, self.access.port, self.max_reconnects, last_err))
def test_no_retry_on_non_timeout_ftp_errors(self): """FTP errors other than timeouts should not trigger a retry""" with mock.patch('ftplib.FTP'): crawler = crawlers.FTPCrawler('ftp://foo') crawler.ftp.nlst.side_effect = ftplib.error_temp('422') with mock.patch.object(crawler, 'connect') as mock_connect: with self.assertRaises(ftplib.error_temp): crawler._list_folder_contents('/') mock_connect.assert_not_called()
def test_translate_directory_not_empty(self): # https://bugs.launchpad.net/bugs/528722 t = ftp.FtpTransport("ftp://none/") try: raise ftplib.error_temp("Rename/move failure: Directory not empty") except Exception, e: e = self.assertRaises(errors.DirectoryNotEmpty, t._translate_ftp_error, e, "/path")
def _rmDirR(dirpath): """Remove the directory at dirpath and its contents (recursive).""" try: dirs, files = listRemote(dirpath); for f in files: _rm(f.path); for d in dirs: _rmDirR(d.path); _rmDir(d.path); except: raise error_temp("451 Can't remove directory");
def test_retry_open(mocker): mocker.patch("peakina.io.ftp.ftp_utils._open").side_effect = [ ftplib.error_temp("421 Could not create socket"), AttributeError("'NoneType' object has no attribute 'sendall'"), OSError("Random OSError"), "ok", ] mock_sleep = mocker.patch("peakina.io.ftp.ftp_utils.sleep") ret = ftp_open(url="foo") calls = [mocker.call(2), mocker.call(8), mocker.call(18)] mock_sleep.assert_has_calls(calls) assert ret == "ok" # type: ignore[comparison-overlap]
def test_retry_open(mocker): mocker.patch('peakina.io.ftp.ftp_utils._open').side_effect = [ ftplib.error_temp('421 Could not create socket'), AttributeError("'NoneType' object has no attribute 'sendall'"), OSError('Random OSError'), 'ok', ] mock_sleep = mocker.patch('peakina.io.ftp.ftp_utils.sleep') ret = ftp_open(url="foo") calls = [mocker.call(2), mocker.call(8), mocker.call(18)] mock_sleep.assert_has_calls(calls) assert ret == 'ok'
def listdir(self, directory='.'): """Will always list the directory, even if a file is given""" from ftplib import error_reply, error_temp try: flist = self.ftp.nlst(directory) return sorted(flist) except error_temp: return [] except BrokenPipeError: raise error_temp( 'Server timeout. Try clossing and restarting the connection') except error_reply: return []
def searchPubmed(searchFields, sortby='relevance', num="10", resultsFormat='json'): """ Searches PubMed database for MeSH terms and other additional fields ('searchFields'), sorts them by relevance and \ returns the top 'num'. :param list searchFields: list of search fields to query for. :param str sortby: parameter to use for sorting. :param str num: number of PubMed identifiers to return. :param str resultsFormat: format of the PubMed result. :return: Dictionary with total number of PubMed ids, and top 'num' ids. """ pubmedQueryUrl = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=TERM&retmode=json&retmax=NUM' if len(searchFields) > 1: query = " [MeSH Terms] AND ".join(searchFields) else: query = searchFields[0] + " [MeSH Terms] AND" try: url = pubmedQueryUrl.replace('TERMS', query).replace('NUM', num) http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) response = http.request("GET", urllib.parse.quote(url)) jsonResponse = response.read() resultDict = json.loads(jsonResponse) except urllib3.exceptions.InvalidHeader: raise urllib3.exceptions.InvalidHeader("Invalid HTTP header provided. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.ConnectTimeoutError: raise urllib3.exceptions.ConnectTimeoutError("Connection timeout requesting URL. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.ConnectionError: raise urllib3.exceptions.ConnectionError("Protocol error when downloading. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.DecodeError: raise urllib3.exceptions.DecodeError("Decoder error when downloading. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.SecurityWarning: raise urllib3.exceptions.SecurityWarning("Security warning when downloading. {}.\nURL:{}".format(err,url)) except urllib3.exceptions.ProtocolError: raise urllib3.exceptions.ProtocolError("Protocol error when downloading. {}.\nURL:{}".format(err,url)) except ftplib.error_reply as err: raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,url)) except ftplib.error_temp as err: raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,url)) except ftplib.error_perm as err: raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,url)) except ftplib.error_proto: raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,url)) except Exception as err: raise Exception("Something went wrong. {}.\nURL:{}".format(err,url)) result = [] if 'esearchresult' in resultDict: result = resultDict['esearchresult'] return result
def test_retry_on_timeout_decorator_timeout_error(self): """The retry_on_timeout decorator should re-create the connection when a FTP timeout error occurs, and and re-run the method in which the error occurred once """ with mock.patch('ftplib.FTP'): crawler = crawlers.FTPCrawler('ftp://foo') crawler.ftp.nlst.side_effect = ftplib.error_temp('421') with self.assertRaises(ftplib.error_temp), \ self.assertLogs(crawler.LOGGER, level=logging.INFO) as log_cm: crawler._list_folder_contents('/') self.assertEqual(log_cm.records[0].getMessage(), "Re-initializing the FTP connection")
def getresp(self): resp = self.getmultiline() if self.debugging: print('*resp*', self.sanitize(resp)) self.lastresp = resp[:3] c = resp[:1] if c in {'1', '2', '3'}: return resp if c == '4': raise ftplib.error_temp(resp) if c == '5': if self.lastresp == '553' and self.use_xdupe: with self.lock: self.parse_xdupe(resp) raise ftplib.error_perm(resp) raise ftplib.error_proto(resp)
def download_from_ftp(ftp_url, user, password, to, file_name): try: domain = ftp_url.split('/')[2] ftp_file = '/'.join(ftp_url.split('/')[3:]) with ftplib.FTP(domain) as ftp: ftp.login(user=user, passwd=password) with open(os.path.join(to, file_name), 'wb') as fp: ftp.retrbinary("RETR " + ftp_file, fp.write) except ftplib.error_reply as err: raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,ftp_url)) except ftplib.error_temp as err: raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,ftp_url)) except ftplib.error_perm as err: raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,ftp_url)) except ftplib.error_proto: raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,ftp_url))
def getresp(self): """ ftplib.getresp : parse JOBNAME in 250/125 z/OS FTP response """ resp = self.getmultiline() if self.debugging: print('*resp*', self.sanitize(resp)) self.lastresp = resp[:3] c = resp[:1] if c in ('1', '2', '3'): if resp[:3] in('250','125'): #|Zftp spec sx0 = re.search(r"\s+(JOB\d{5})\s+", resp) #| if sx0: #| self.__jobid = sx0.group(1) #| return resp if c == '4': raise ftplib.error_temp(resp) if c == '5': raise ftplib.error_perm(resp) raise ftplib.error_proto(resp)
def sendfile_fail_test(self): import log_picker.sending.ftpsender as ftp_s ftp_s.open = self.fs.open ftp_s.ftplib = mock.Mock() ftp_s.os = mock.Mock() ftp_s.file = mock.Mock() import ftplib ftp_s.ftplib = mock.Mock() ftp_s.ftplib.all_errors = ftplib.all_errors ftp_s.ftplib.FTP().storbinary.side_effect = ftplib.error_temp("some") HOST = "localhost" FILE = "/tmp/somefile" MIMETYPE = "application/x-bzip2" self.fs.open(FILE, 'w').write("some content") obj = ftp_s.FtpSender() obj.set_host(HOST) self.assertRaises(ftp_s.SenderError, obj.sendfile, FILE, MIMETYPE)
def _with_reconnects(self, func, *args, **kwargs): """Try and execute a function, reconnecting on failure.""" for _ in range(self.max_reconnects): try: self._connect() return func(*args, **kwargs) except ( ConnectionRefusedError, ConnectionResetError, socket.timeout, socket.gaierror, socket.herror, error_temp, error_perm, EOFError, OSError, ) as err: self.quit() last_err = err raise error_temp(f"Failed after {self.max_reconnects} reconnect(s), " f"the last error was: {last_err}")
def timed_out_pwd(): raise ftplib.error_temp("simulated timeout")
def downloadDB(databaseURL, directory=None, file_name=None, user="", password=""): """ This function downloads the raw files from a biomedical database server when a link is provided. :param str databaseURL: link to access biomedical database server. :param directory: :type directory: str or None :param file_name: name of the file to dowload. If None, 'databaseURL' must contain \ filename after the last '/'. :type file_name: str or None :param str user: username to access biomedical database server if required. :param str password: password to access biomedical database server if required. """ dbconfig = setup_config() if directory is None: directory = dbconfig["databasesDir"] if file_name is None: file_name = databaseURL.split('/')[-1].replace('?', '_').replace('=', '_') header = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36' } try: mode = 'wb' if databaseURL.startswith('ftp:'): domain = databaseURL.split('/')[2] ftp_file = '/'.join(databaseURL.split('/')[3:]) with ftplib.FTP(domain) as ftp: ftp.login(user=user, passwd=password) ftp.retrbinary( "RETR " + ftp_file, open(os.path.join(directory, file_name), mode).write) else: if os.path.exists(os.path.join(directory, file_name)): os.remove(os.path.join(directory, file_name)) try: wget.download(databaseURL, os.path.join(directory, file_name)) except Exception: r = requests.get(databaseURL, headers=header) with open(os.path.join(directory, file_name), 'wb') as out: out.write(r.content) #os.system("wget -O {0} {1}".format(os.path.join(directory, file_name), databaseURL)) except ftplib.error_reply as err: raise ftplib.error_reply( "Exception raised when an unexpected reply is received from the server. {}.\nURL:{}" .format(err, databaseURL)) except ftplib.error_temp as err: raise ftplib.error_temp( "Exception raised when an error code signifying a temporary error. {}.\nURL:{}" .format(err, databaseURL)) except ftplib.error_perm as err: raise ftplib.error_perm( "Exception raised when an error code signifying a permanent error. {}.\nURL:{}" .format(err, databaseURL)) except ftplib.error_proto: raise ftplib.error_proto( "Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}" .format(err, databaseURL)) except Exception as err: raise Exception("Something went wrong. {}.\nURL:{}".format( err, databaseURL))