def ntransfercmd(self, cmd, rest=None):
size = None
if self.passiveserver:
host, port = self.makepasv()
conn = socket.create_connection((host, port), self.timeout,
self.source_address)
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
except:
conn.close()
raise
else:
sock = self.makeport()
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
conn, sockaddr = sock.accept()
if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
conn.settimeout(self.timeout)
finally:
sock.close()
if resp[:3] == '150':
size = ftplib.parse150(resp)
return conn, size
def ntransfercmd(self, cmd, rest=None):
size = None
if self.passiveserver:
host, port = self.makepasv()
conn = socket.create_connection((host, port), self.timeout,
self.source_address)
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
except Exception:
conn.close()
raise
else:
sock = self.makeport()
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
conn, sockaddr = sock.accept()
if self.timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
conn.settimeout(self.timeout)
finally:
sock.close()
if resp[:3] == '150':
size = ftplib.parse150(resp)
return conn, size
def ntransfercmd(self, cmd, rest=None):
# Simulate the real ntransfercmd. We don't need to special-case passive
# as we use our own data channels in any event.
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
return (self.data_channels.pop(0), None)
def get_stats(f, ftp):
resp = ftp.sendcmd('MDTM %s' %f) # Ask for file's modification time
# parse response into a datetime if response code is 213 - OK file status.
if resp[:3] == '213':
# substract 6 from hour becuase tzinfo is too complicated and would require pytz
if int(resp[12:14])-6 < 0:
fmtime = datetime(int(resp[4:8]), int(resp[8:10]), int(resp[10:12])-1, (int(resp[12:14])-6)%24, int(resp[14:16]), int(resp[16:18]))
else:
fmtime = datetime(int(resp[4:8]), int(resp[8:10]), int(resp[10:12]), int(resp[12:14])-6, int(resp[14:16]), int(resp[16:18]))
return {'mtime': fmtime, 'size': ftp.size(f)}
else:
raise error_reply('Error FTP: Respuesta insperada')
def searchPubmed(searchFields, sortby='relevance', num="10", resultsFormat='json'):
"""
Searches PubMed database for MeSH terms and other additional fields ('searchFields'), sorts them by relevance and \
returns the top 'num'.
:param list searchFields: list of search fields to query for.
:param str sortby: parameter to use for sorting.
:param str num: number of PubMed identifiers to return.
:param str resultsFormat: format of the PubMed result.
:return: Dictionary with total number of PubMed ids, and top 'num' ids.
"""
pubmedQueryUrl = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=TERM&retmode=json&retmax=NUM'
if len(searchFields) > 1:
query = " [MeSH Terms] AND ".join(searchFields)
else:
query = searchFields[0] + " [MeSH Terms] AND"
try:
url = pubmedQueryUrl.replace('TERMS', query).replace('NUM', num)
http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())
response = http.request("GET", urllib.parse.quote(url))
jsonResponse = response.read()
resultDict = json.loads(jsonResponse)
except urllib3.exceptions.InvalidHeader:
raise urllib3.exceptions.InvalidHeader("Invalid HTTP header provided. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.ConnectTimeoutError:
raise urllib3.exceptions.ConnectTimeoutError("Connection timeout requesting URL. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.ConnectionError:
raise urllib3.exceptions.ConnectionError("Protocol error when downloading. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.DecodeError:
raise urllib3.exceptions.DecodeError("Decoder error when downloading. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.SecurityWarning:
raise urllib3.exceptions.SecurityWarning("Security warning when downloading. {}.\nURL:{}".format(err,url))
except urllib3.exceptions.ProtocolError:
raise urllib3.exceptions.ProtocolError("Protocol error when downloading. {}.\nURL:{}".format(err,url))
except ftplib.error_reply as err:
raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,url))
except ftplib.error_temp as err:
raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,url))
except ftplib.error_perm as err:
raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,url))
except ftplib.error_proto:
raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,url))
except Exception as err:
raise Exception("Something went wrong. {}.\nURL:{}".format(err,url))
result = []
if 'esearchresult' in resultDict:
result = resultDict['esearchresult']
return result
def download_from_ftp(ftp_url, user, password, to, file_name):
try:
domain = ftp_url.split('/')[2]
ftp_file = '/'.join(ftp_url.split('/')[3:])
with ftplib.FTP(domain) as ftp:
ftp.login(user=user, passwd=password)
with open(os.path.join(to, file_name), 'wb') as fp:
ftp.retrbinary("RETR " + ftp_file, fp.write)
except ftplib.error_reply as err:
raise ftplib.error_reply("Exception raised when an unexpected reply is received from the server. {}.\nURL:{}".format(err,ftp_url))
except ftplib.error_temp as err:
raise ftplib.error_temp("Exception raised when an error code signifying a temporary error. {}.\nURL:{}".format(err,ftp_url))
except ftplib.error_perm as err:
raise ftplib.error_perm("Exception raised when an error code signifying a permanent error. {}.\nURL:{}".format(err,ftp_url))
except ftplib.error_proto:
raise ftplib.error_proto("Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}".format(err,ftp_url))
def register(self, user, passwd, acct=''):
"""
Register a new user. This works similarly to login but starts with an RGTR call instead of USER.
The username is encrypted and the server password is derived from the given password.
:param user: (str) username
:param passwd: (str) secret
:param acct: [unused]
:return: (str) server response
"""
self._cipher = MyCipher(passwd)
user = self._encrypt_filename(user)
resp = self.sendcmd('RGTR ' + user)
if resp[0] == '3':
server_key = self._cipher.derive_server_key()
resp = self.sendcmd('PASS ' + server_key)
if resp[0] == '3':
resp = self.sendcmd('ACCT ' + acct)
if resp[0] != '2':
raise error_reply(resp)
return resp
def ntransfercmd(self, cmd, rest=None):
assert self.passiveserver
host, port = self.makepasv()
conn = gevent.socket.create_connection((host, port), self.timeout)
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
except:
conn.close()
raise
if resp[:3] == '150':
size = ftplib.parse150(resp)
return conn, size
def ntransfercmd(self, cmd, rest=None):
assert self.passiveserver
host, port = self.makepasv()
conn = gevent.socket.create_connection((host, port), self.timeout)
try:
if rest is not None:
self.sendcmd("REST %s" % rest)
resp = self.sendcmd(cmd)
if resp[0] == '2':
resp = self.getresp()
if resp[0] != '1':
raise ftplib.error_reply(resp)
except:
conn.close()
raise
if resp[:3] == '150':
size = ftplib.parse150(resp)
return conn, size
def login(self, user="", password="", acct=""):
"""
Public method to login to the FTP server.
This extended method respects the FTP proxy configuration. There are
many different FTP proxy products available. But unfortunately there
is no standard for how o traverse a FTP proxy. The lis below shows
the sequence of commands used.
<table>
<tr><td>user</td><td>Username for remote host</td></tr>
<tr><td>pass</td><td>Password for remote host</td></tr>
<tr><td>pruser</td><td>Username for FTP proxy</td></tr>
<tr><td>prpass</td><td>Password for FTP proxy</td></tr>
<tr><td>remote.host</td><td>Hostname of the remote FTP server</td>
</tr>
</table>
<dl>
<dt>E5FtpProxyType.NoProxy:</dt>
<dd>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.NonAuthorizing:</dt>
<dd>
USER [email protected]<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.UserAtServer:</dt>
<dd>
USER pruser<br/>
PASS prpass<br/>
USER [email protected]<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.Site:</dt>
<dd>
USER pruser<br/>
PASS prpass<br/>
SITE remote.site<br/>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.Open:</dt>
<dd>
USER pruser<br/>
PASS prpass<br/>
OPEN remote.site<br/>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.UserAtProxyuserAtServer:</dt>
<dd>
USER user@[email protected]<br/>
PASS pass@prpass
</dd>
<dt>E5FtpProxyType.ProxyuserAtServer:</dt>
<dd>
USER [email protected]<br/>
PASS prpass<br/>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.AuthResp:</dt>
<dd>
USER [email protected]<br/>
PASS pass<br/>
AUTH pruser<br/>
RESP prpass
</dd>
<dt>E5FtpProxyType.Bluecoat:</dt>
<dd>
USER [email protected] pruser<br/>
PASS pass<br/>
ACCT prpass
</dd>
</dl>
@param user username for the remote host (string)
@param password password for the remote host (string)
@param acct accounting information for the remote host (string)
@return response sent by the remote host (string)
@exception E5FtpProxyError raised to indicate a proxy related issue
@exception ftplib.error_reply raised to indicate an FTP error reply
"""
if not user:
user = "******"
if not password:
# make sure it is a string
password = ""
if not acct:
# make sure it is a string
acct = ""
if user == "anonymous" and password in {'', '-'}:
password += "anonymous@"
if self.__proxyType != E5FtpProxyType.NoProxy:
if self.__proxyType != E5FtpProxyType.NonAuthorizing:
# check, if a valid proxy configuration is known
if not self.__proxyUser:
raise E5FtpProxyError(
"991 Proxy usage requested, but no proxy user given")
if not self.__proxyPassword:
raise E5FtpProxyError(
"992 Proxy usage requested, but no proxy password"
" given")
if self.__proxyType in [E5FtpProxyType.NonAuthorizing,
E5FtpProxyType.AuthResp,
E5FtpProxyType.Bluecoat]:
user += "@" + self.__host
if self.__proxyType == E5FtpProxyType.Bluecoat:
user += " " + self.__proxyUser
acct = self.__proxyPassword
elif self.__proxyType == E5FtpProxyType.UserAtProxyuserAtServer:
user = "******".format(
user, self.__proxyUser, self.__host)
password = "******".format(password, self.__proxyPassword)
else:
pruser = self.__proxyUser
if self.__proxyType == E5FtpProxyType.UserAtServer:
user += "@" + self.__host
elif self.__proxyType == E5FtpProxyType.ProxyuserAtServer:
pruser += "@" + self.__host
# authenticate to the proxy first
presp = self.sendcmd("USER " + pruser)
if presp[0] == "3":
presp = self.sendcmd("PASS " + self.__proxyPassword)
if presp[0] == "3" and self.__proxyAccount:
presp = self.sendcmd("ACCT " + self.__proxyAccount)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error authorizing at proxy\n{1}".format(
presp[0], presp))
if self.__proxyType == E5FtpProxyType.Site:
# send SITE command
presp = self.sendcmd("SITE " + self.__host)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error sending SITE command\n{1}".format(
presp[0], presp))
elif self.__proxyType == E5FtpProxyType.Open:
# send OPEN command
presp = self.sendcmd("OPEN " + self.__host)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error sending OPEN command\n{1}".format(
presp[0], presp))
# authenticate to the remote host or combined to proxy and remote host
resp = self.sendcmd("USER " + user)
if resp[0] == "3":
resp = self.sendcmd("PASS " + password)
if resp[0] == "3":
resp = self.sendcmd("ACCT " + acct)
if resp[0] != "2":
raise ftplib.error_reply(resp)
if self.__proxyType == E5FtpProxyType.AuthResp:
# authorize to the FTP proxy
presp = self.sendcmd("AUTH " + self.__proxyUser)
if presp[0] == "3":
presp = self.sendcmd("RESP " + self.__proxyPassword)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error authorizing at proxy\n{1}".format(
presp[0], presp))
return resp
def timed_out_pwd():
raise ftplib.error_reply("delayed 226 reply")
def login(self):
"""Login to FTP server."""
self.ftp = FTP(self.FTP_HOST)
self.ftp.login()
if self.ftp.lastresp != "230":
raise error_reply("FTP login error.")
def login(self, user="", password="", acct=""):
"""
Public method to login to the FTP server.
This extended method respects the FTP proxy configuration. There are
many different FTP proxy products available. But unfortunately there
is no standard for how o traverse a FTP proxy. The lis below shows
the sequence of commands used.
<table>
<tr><td>user</td><td>Username for remote host</td></tr>
<tr><td>pass</td><td>Password for remote host</td></tr>
<tr><td>pruser</td><td>Username for FTP proxy</td></tr>
<tr><td>prpass</td><td>Password for FTP proxy</td></tr>
<tr><td>remote.host</td><td>Hostname of the remote FTP server</td>
</tr>
</table>
<dl>
<dt>E5FtpProxyType.NoProxy:</dt>
<dd>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.NonAuthorizing:</dt>
<dd>
USER [email protected]<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.UserAtServer:</dt>
<dd>
USER pruser<br/>
PASS prpass<br/>
USER [email protected]<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.Site:</dt>
<dd>
USER pruser<br/>
PASS prpass<br/>
SITE remote.site<br/>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.Open:</dt>
<dd>
USER pruser<br/>
PASS prpass<br/>
OPEN remote.site<br/>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.UserAtProxyuserAtServer:</dt>
<dd>
USER user@[email protected]<br/>
PASS pass@prpass
</dd>
<dt>E5FtpProxyType.ProxyuserAtServer:</dt>
<dd>
USER [email protected]<br/>
PASS prpass<br/>
USER user<br/>
PASS pass
</dd>
<dt>E5FtpProxyType.AuthResp:</dt>
<dd>
USER [email protected]<br/>
PASS pass<br/>
AUTH pruser<br/>
RESP prpass
</dd>
<dt>E5FtpProxyType.Bluecoat:</dt>
<dd>
USER [email protected] pruser<br/>
PASS pass<br/>
ACCT prpass
</dd>
</dl>
@param user username for the remote host (string)
@param password password for the remote host (string)
@param acct accounting information for the remote host (string)
@return response sent by the remote host (string)
@exception E5FtpProxyError raised to indicate a proxy related issue
"""
if not user:
user = "******"
if not password:
# make sure it is a string
password = ""
if not acct:
# make sure it is a string
acct = ""
if user == "anonymous" and password in {'', '-'}:
password += "anonymous@"
if self.__proxyType != E5FtpProxyType.NoProxy:
if self.__proxyType != E5FtpProxyType.NonAuthorizing:
# check, if a valid proxy configuration is known
if not self.__proxyUser:
raise E5FtpProxyError(
"991 Proxy usage requested, but no proxy user given")
if not self.__proxyPassword:
raise E5FtpProxyError(
"992 Proxy usage requested, but no proxy password"
" given")
if self.__proxyType in [E5FtpProxyType.NonAuthorizing,
E5FtpProxyType.AuthResp,
E5FtpProxyType.Bluecoat]:
user += "@" + self.__host
if self.__proxyType == E5FtpProxyType.Bluecoat:
user += " " + self.__proxyUser
acct = self.__proxyPassword
elif self.__proxyType == E5FtpProxyType.UserAtProxyuserAtServer:
user = "******".format(
user, self.__proxyUser, self.__host)
password = "******".format(password, self.__proxyPassword)
else:
pruser = self.__proxyUser
if self.__proxyType == E5FtpProxyType.UserAtServer:
user += "@" + self.__host
elif self.__proxyType == E5FtpProxyType.ProxyuserAtServer:
pruser += "@" + self.__host
# authenticate to the proxy first
presp = self.sendcmd("USER " + pruser)
if presp[0] == "3":
presp = self.sendcmd("PASS " + self.__proxyPassword)
if presp[0] == "3" and self.__proxyAccount:
presp = self.sendcmd("ACCT " + self.__proxyAccount)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error authorizing at proxy\n{1}".format(
presp[0], presp))
if self.__proxyType == E5FtpProxyType.Site:
# send SITE command
presp = self.sendcmd("SITE " + self.__host)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error sending SITE command\n{1}".format(
presp[0], presp))
elif self.__proxyType == E5FtpProxyType.Open:
# send OPEN command
presp = self.sendcmd("OPEN " + self.__host)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error sending OPEN command\n{1}".format(
presp[0], presp))
# authenticate to the remote host or combined to proxy and remote host
resp = self.sendcmd("USER " + user)
if resp[0] == "3":
resp = self.sendcmd("PASS " + password)
if resp[0] == "3":
resp = self.sendcmd("ACCT " + acct)
if resp[0] != "2":
raise ftplib.error_reply(resp)
if self.__proxyType == E5FtpProxyType.AuthResp:
# authorize to the FTP proxy
presp = self.sendcmd("AUTH " + self.__proxyUser)
if presp[0] == "3":
presp = self.sendcmd("RESP " + self.__proxyPassword)
if presp[0] != "2":
raise E5FtpProxyError(
"9{0}0 Error authorizing at proxy\n{1}".format(
presp[0], presp))
return resp
def downloadDB(databaseURL,
directory=None,
file_name=None,
user="",
password=""):
"""
This function downloads the raw files from a biomedical database server when a link is provided.
:param str databaseURL: link to access biomedical database server.
:param directory:
:type directory: str or None
:param file_name: name of the file to dowload. If None, 'databaseURL' must contain \
filename after the last '/'.
:type file_name: str or None
:param str user: username to access biomedical database server if required.
:param str password: password to access biomedical database server if required.
"""
dbconfig = setup_config()
if directory is None:
directory = dbconfig["databasesDir"]
if file_name is None:
file_name = databaseURL.split('/')[-1].replace('?',
'_').replace('=', '_')
header = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36'
}
try:
mode = 'wb'
if databaseURL.startswith('ftp:'):
domain = databaseURL.split('/')[2]
ftp_file = '/'.join(databaseURL.split('/')[3:])
with ftplib.FTP(domain) as ftp:
ftp.login(user=user, passwd=password)
ftp.retrbinary(
"RETR " + ftp_file,
open(os.path.join(directory, file_name), mode).write)
else:
if os.path.exists(os.path.join(directory, file_name)):
os.remove(os.path.join(directory, file_name))
try:
wget.download(databaseURL, os.path.join(directory, file_name))
except Exception:
r = requests.get(databaseURL, headers=header)
with open(os.path.join(directory, file_name), 'wb') as out:
out.write(r.content)
#os.system("wget -O {0} {1}".format(os.path.join(directory, file_name), databaseURL))
except ftplib.error_reply as err:
raise ftplib.error_reply(
"Exception raised when an unexpected reply is received from the server. {}.\nURL:{}"
.format(err, databaseURL))
except ftplib.error_temp as err:
raise ftplib.error_temp(
"Exception raised when an error code signifying a temporary error. {}.\nURL:{}"
.format(err, databaseURL))
except ftplib.error_perm as err:
raise ftplib.error_perm(
"Exception raised when an error code signifying a permanent error. {}.\nURL:{}"
.format(err, databaseURL))
except ftplib.error_proto:
raise ftplib.error_proto(
"Exception raised when a reply is received from the server that does not fit the response specifications of the File Transfer Protocol. {}.\nURL:{}"
.format(err, databaseURL))
except Exception as err:
raise Exception("Something went wrong. {}.\nURL:{}".format(
err, databaseURL))
