def __wrap_reconnect(self, method: Callable[[ftplib.FTP], T]) -> T:
"""
Dispatche the method to the connection and reconnects if needed.
:param method: to be dispatched
:return: response from the method
"""
last_err = None # type: Optional[Exception]
for _ in range(0, self.max_reconnects):
try:
if self.connection is None:
self.connect()
assert self.connection is not None, "Expected connect() to either raise or create a connection"
return method(self.connection)
except (ConnectionRefusedError, ConnectionAbortedError,
socket.timeout, socket.gaierror, socket.herror,
ftplib.error_temp, EOFError) as err:
self.connection.close()
self.connection = None
last_err = err
assert last_err is not None, 'Expected either an error or a previous return'
raise ftplib.error_temp(
"Failed to execute a command on {}:{} after {} reconnect(s), the last error was: {}"
.format(self.access.hostname, self.access.port,
self.max_reconnects, last_err))
def test_no_retry_on_non_timeout_ftp_errors(self):
"""FTP errors other than timeouts should not trigger a retry"""
with mock.patch('ftplib.FTP'):
crawler = crawlers.FTPCrawler('ftp://foo')
crawler.ftp.nlst.side_effect = ftplib.error_temp('422')
with mock.patch.object(crawler, 'connect') as mock_connect:
with self.assertRaises(ftplib.error_temp):
crawler._list_folder_contents('/')
mock_connect.assert_not_called()
def test_translate_directory_not_empty(self):
# https://bugs.launchpad.net/bugs/528722
t = ftp.FtpTransport("ftp://none/")
try:
raise ftplib.error_temp("Rename/move failure: Directory not empty")
except Exception, e:
e = self.assertRaises(errors.DirectoryNotEmpty,
t._translate_ftp_error, e, "/path")
def _rmDirR(dirpath):
"""Remove the directory at dirpath and its contents (recursive)."""
try:
dirs, files = listRemote(dirpath);
for f in files:
_rm(f.path);
for d in dirs:
_rmDirR(d.path);
_rmDir(d.path);
except:
raise error_temp("451 Can't remove directory");
def test_retry_open(mocker):
mocker.patch("peakina.io.ftp.ftp_utils._open").side_effect = [
ftplib.error_temp("421 Could not create socket"),
AttributeError("'NoneType' object has no attribute 'sendall'"),
OSError("Random OSError"),
"ok",
]
mock_sleep = mocker.patch("peakina.io.ftp.ftp_utils.sleep")
ret = ftp_open(url="foo")
calls = [mocker.call(2), mocker.call(8), mocker.call(18)]
mock_sleep.assert_has_calls(calls)
assert ret == "ok" # type: ignore[comparison-overlap]
def test_retry_open(mocker):
mocker.patch('peakina.io.ftp.ftp_utils._open').side_effect = [
ftplib.error_temp('421 Could not create socket'),
AttributeError("'NoneType' object has no attribute 'sendall'"),
OSError('Random OSError'),
'ok',
]
mock_sleep = mocker.patch('peakina.io.ftp.ftp_utils.sleep')
ret = ftp_open(url="foo")
calls = [mocker.call(2), mocker.call(8), mocker.call(18)]
mock_sleep.assert_has_calls(calls)
assert ret == 'ok'
def listdir(self, directory='.'):
"""Will always list the directory, even if a file is given"""
from ftplib import error_reply, error_temp
try:
flist = self.ftp.nlst(directory)
return sorted(flist)
except error_temp:
return []
except BrokenPipeError:
raise error_temp(
'Server timeout. Try clossing and restarting the connection')
except error_reply:
return []
def searchPubmed(searchFields, sortby='relevance', num="10", resultsFormat='json'):
"""
Searches PubMed database for MeSH terms and other additional fields ('searchFields'), sorts them by relevance and \
returns the top 'num'.
:param list searchFields: list of search fields to query for.
:param str sortby: parameter to use for sorting.
:param str num: number of PubMed identifiers to return.
:param str resultsFormat: format of the PubMed result.
:return: Dictionary with total number of PubMed ids, and top 'num' ids.
"""
pubmedQueryUrl = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=TERM&retmode=json&retmax=NUM'
if len(searchFields) > 1:
query = " [MeSH Terms] AND ".join(searchFields)
else:
query = searchFields[0] + " [MeSH Terms] AND"
try:
url = pubmedQueryUrl.replace('TERMS', query).replace('NUM', num)
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
response = http.request("GET", urllib.parse.quote(url))
jsonResponse = response.read()
resultDict = json.loads(jsonResponse)
except urllib3.exceptions.InvalidHeader:
raise urllib3.exceptions.InvalidHeader("Invalid HTTP header provided. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.ConnectTimeoutError:
raise urllib3.exceptions.ConnectTimeoutError("Connection timeout requesting URL. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.ConnectionError:
raise urllib3.exceptions.ConnectionError("Protocol error when downloading. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.DecodeError:
raise urllib3.exceptions.DecodeError("Decoder error when downloading. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.SecurityWarning:
raise urllib3.exceptions.SecurityWarning("Security warning when downloading. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.ProtocolError:
raise urllib3.exceptions.ProtocolError("Protocol error when downloading. {}.\nURL:{}".format(err,url))
except ftplib.error_reply as err:
raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,url))
except ftplib.error_temp as err:
raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,url))
except ftplib.error_perm as err:
raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,url))
except ftplib.error_proto:
raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,url))
except Exception as err:
raise Exception("Something went wrong. {}.\nURL:{}".format(err,url))
result = []
if 'esearchresult' in resultDict:
result = resultDict['esearchresult']
return result
def test_retry_on_timeout_decorator_timeout_error(self):
"""The retry_on_timeout decorator should re-create
the connection when a FTP timeout error occurs, and
and re-run the method in which the error occurred once
"""
with mock.patch('ftplib.FTP'):
crawler = crawlers.FTPCrawler('ftp://foo')
crawler.ftp.nlst.side_effect = ftplib.error_temp('421')
with self.assertRaises(ftplib.error_temp), \
self.assertLogs(crawler.LOGGER, level=logging.INFO) as log_cm:
crawler._list_folder_contents('/')
self.assertEqual(log_cm.records[0].getMessage(),
"Re-initializing the FTP connection")
def getresp(self):
resp = self.getmultiline()
if self.debugging:
print('*resp*', self.sanitize(resp))
self.lastresp = resp[:3]
c = resp[:1]
if c in {'1', '2', '3'}:
return resp
if c == '4':
raise ftplib.error_temp(resp)
if c == '5':
if self.lastresp == '553' and self.use_xdupe:
with self.lock:
self.parse_xdupe(resp)
raise ftplib.error_perm(resp)
raise ftplib.error_proto(resp)
def download_from_ftp(ftp_url, user, password, to, file_name):
try:
domain = ftp_url.split('/')[2]
ftp_file = '/'.join(ftp_url.split('/')[3:])
with ftplib.FTP(domain) as ftp:
ftp.login(user=user, passwd=password)
with open(os.path.join(to, file_name), 'wb') as fp:
ftp.retrbinary("RETR " + ftp_file, fp.write)
except ftplib.error_reply as err:
raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,ftp_url))
except ftplib.error_temp as err:
raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,ftp_url))
except ftplib.error_perm as err:
raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,ftp_url))
except ftplib.error_proto:
raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,ftp_url))
def getresp(self):
"""
ftplib.getresp :
parse JOBNAME in 250/125 z/OS FTP response
"""
resp = self.getmultiline()
if self.debugging:
print('*resp*', self.sanitize(resp))
self.lastresp = resp[:3]
c = resp[:1]
if c in ('1', '2', '3'):
if resp[:3] in('250','125'): #|Zftp spec
sx0 = re.search(r"\s+(JOB\d{5})\s+", resp) #|
if sx0: #|
self.__jobid = sx0.group(1) #|
return resp
if c == '4':
raise ftplib.error_temp(resp)
if c == '5':
raise ftplib.error_perm(resp)
raise ftplib.error_proto(resp)
def sendfile_fail_test(self):
import log_picker.sending.ftpsender as ftp_s
ftp_s.open = self.fs.open
ftp_s.ftplib = mock.Mock()
ftp_s.os = mock.Mock()
ftp_s.file = mock.Mock()
import ftplib
ftp_s.ftplib = mock.Mock()
ftp_s.ftplib.all_errors = ftplib.all_errors
ftp_s.ftplib.FTP().storbinary.side_effect = ftplib.error_temp("some")
HOST = "localhost"
FILE = "/tmp/somefile"
MIMETYPE = "application/x-bzip2"
self.fs.open(FILE, 'w').write("some content")
obj = ftp_s.FtpSender()
obj.set_host(HOST)
self.assertRaises(ftp_s.SenderError, obj.sendfile, FILE, MIMETYPE)
def sendfile_fail_test(self):
import log_picker.sending.ftpsender as ftp_s
ftp_s.open = self.fs.open
ftp_s.ftplib = mock.Mock()
ftp_s.os = mock.Mock()
ftp_s.file = mock.Mock()
import ftplib
ftp_s.ftplib = mock.Mock()
ftp_s.ftplib.all_errors = ftplib.all_errors
ftp_s.ftplib.FTP().storbinary.side_effect = ftplib.error_temp("some")
HOST = "localhost"
FILE = "/tmp/somefile"
MIMETYPE = "application/x-bzip2"
self.fs.open(FILE, 'w').write("some content")
obj = ftp_s.FtpSender()
obj.set_host(HOST)
self.assertRaises(ftp_s.SenderError, obj.sendfile, FILE, MIMETYPE)
def _with_reconnects(self, func, *args, **kwargs):
"""Try and execute a function, reconnecting on failure."""
for _ in range(self.max_reconnects):
try:
self._connect()
return func(*args, **kwargs)
except (
ConnectionRefusedError,
ConnectionResetError,
socket.timeout,
socket.gaierror,
socket.herror,
error_temp,
error_perm,
EOFError,
OSError,
) as err:
self.quit()
last_err = err
raise error_temp(f"Failed after {self.max_reconnects} reconnect(s), "
f"the last error was: {last_err}")
def timed_out_pwd():
raise ftplib.error_temp("simulated timeout")
def timed_out_pwd():
raise ftplib.error_temp("simulated timeout")
def downloadDB(databaseURL,
directory=None,
file_name=None,
user="",
password=""):
"""
This function downloads the raw files from a biomedical database server when a link is provided.
:param str databaseURL: link to access biomedical database server.
:param directory:
:type directory: str or None
:param file_name: name of the file to dowload. If None, 'databaseURL' must contain \
filename after the last '/'.
:type file_name: str or None
:param str user: username to access biomedical database server if required.
:param str password: password to access biomedical database server if required.
"""
dbconfig = setup_config()
if directory is None:
directory = dbconfig["databasesDir"]
if file_name is None:
file_name = databaseURL.split('/')[-1].replace('?',
'_').replace('=', '_')
header = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
}
try:
mode = 'wb'
if databaseURL.startswith('ftp:'):
domain = databaseURL.split('/')[2]
ftp_file = '/'.join(databaseURL.split('/')[3:])
with ftplib.FTP(domain) as ftp:
ftp.login(user=user, passwd=password)
ftp.retrbinary(
"RETR " + ftp_file,
open(os.path.join(directory, file_name), mode).write)
else:
if os.path.exists(os.path.join(directory, file_name)):
os.remove(os.path.join(directory, file_name))
try:
wget.download(databaseURL, os.path.join(directory, file_name))
except Exception:
r = requests.get(databaseURL, headers=header)
with open(os.path.join(directory, file_name), 'wb') as out:
out.write(r.content)
#os.system("wget -O {0} {1}".format(os.path.join(directory, file_name), databaseURL))
except ftplib.error_reply as err:
raise ftplib.error_reply(
"Exception raised when an unexpected reply is received from the server. {}.\nURL:{}"
.format(err, databaseURL))
except ftplib.error_temp as err:
raise ftplib.error_temp(
"Exception raised when an error code signifying a temporary error. {}.\nURL:{}"
.format(err, databaseURL))
except ftplib.error_perm as err:
raise ftplib.error_perm(
"Exception raised when an error code signifying a permanent error. {}.\nURL:{}"
.format(err, databaseURL))
except ftplib.error_proto:
raise ftplib.error_proto(
"Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}"
.format(err, databaseURL))
except Exception as err:
raise Exception("Something went wrong. {}.\nURL:{}".format(
err, databaseURL))
