def test_http_list_dateregexp(self):
#self.http_parse.file_date_format = "%%d-%%b-%%Y %%H:%%M"
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
self.assertTrue(len(file_list) == 1)
def test_http_download_in_subdir(self):
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^dists/README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_ftps_list_no_ssl(self):
# This server is misconfigured hence we disable all SSL verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo-user:demo-user"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
def test_download_in_subdir(self):
ftpd = CurlDownload('ftp', 'ftp.fr.debian.org', '/debian/')
(file_list, dir_list) = ftpd.list()
try:
ftpd.match([r'^doc/mailing-lists.txt$'], file_list, dir_list)
except Exception as e:
print("Error: " + str(e))
self.skipTest("Skipping test due to remote server error")
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download(self):
self.utils = UtilsForTest()
self.http_parse = HTTPParse(
"<a[\s]+href=\"([\w\-\.]+\">[\w\-\.]+.tar.gz)<\/a>[\s]+([0-9]{2}-[A-Za-z]{3}-[0-9]{4}[\s][0-9]{2}:[0-9]{2})[\s]+([0-9]+[A-Za-z])",
"<a[\s]+href=\"[\w\-\.]+\">([\w\-\.]+.tar.gz)<\/a>[\s]+([0-9]{2}-[A-Za-z]{3}-[0-9]{4}[\s][0-9]{2}:[0-9]{2})[\s]+([0-9]+[A-Za-z])",
1, 2, 1, 2, None, 3)
self.http_parse.file_date_format = "%%d-%%b-%%Y %%H:%%M"
httpd = CurlDownload('https', 'mirrors.edge.kernel.org',
'/pub/software/scm/git/debian/', self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^git-core-0.99.6.tar.gz$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_download(self):
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
# This tests fails because the zip file is fake. We intercept the failure
# and continue.
# See test_download_skip_check_uncompress
try:
ftpd.download(self.utils.data_dir)
except Exception:
self.assertTrue(1 == 1)
else:
# In case it works, this is the real assertion
self.assertTrue(len(ftpd.files_to_download) == 2)
ftpd.close()
def test_http_download_no_date(self):
self.http_parse = HTTPParse(
self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')), -1,
self.config.get('http.group.file.date_format', None),
int(self.config.get('http.group.file.size')))
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_download(self):
ftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^readme.txt$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_ftps_list(self):
ftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) == 1)
def test_download_ftp_method(self):
"""
Test setting ftp_method (it probably doesn't change anything here but we
test that there is no obvious mistake in the code).
"""
ftpd = CurlDownload("ftp", "test.rebex.net", "/")
ftpd.set_options(dict(ftp_method="nocwd"))
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match(["^readme.txt$"], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download_or_copy(self):
ftpd = CurlDownload('ftp', 'ftp.fr.debian.org', '/debian/')
ftpd.files_to_download = [{
'name': '/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test2',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test11',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}]
available_files = [{
'name': '/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test12',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test3',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 20
}, {
'name': '/test/test11',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}]
ftpd.download_or_copy(available_files, '/biomaj', False)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
self.assertTrue(len(ftpd.files_to_copy) == 2)
def test_http_list(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
self.assertTrue(len(file_list) == 1)
def test_ftp_list(self):
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
def test_download_no_ssl(self):
# This server is misconfigured hence we disable all SSL verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo-user:demo-user"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download_skip_check_uncompress(self):
# This test is similar to test_download but we skip test of zip file.
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
ftpd.set_options(dict(skip_check_uncompress=True))
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
def test_download_ssl_certficate(self):
# This server is misconfigured but we use its certificate
# The hostname is wrong so we disable host verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo-user:demo-user"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
curdir = os.path.dirname(os.path.realpath(__file__))
cert_file = os.path.join(curdir, "caert.demo.wftpserver.com.pem")
ftpd.set_options(
dict(ssl_verifyhost="False", ssl_server_cert=cert_file))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def get_handler(self,
protocol_name,
server,
remote_dir,
remote_files=[],
credentials=None,
http_parse=None,
http_method=None,
param=None,
proxy=None,
proxy_auth='',
save_as=None,
timeout_download=None,
offline_dir=None,
options={}):
protocol = downmessage_pb2.DownloadFile.Protocol.Value(
protocol_name.upper())
downloader = None
if protocol in [0, 1]: # FTP, SFTP
downloader = CurlDownload(protocol_name, server, remote_dir)
if protocol in [
2, 3
]: # HTTP, HTTPS (could be factored with previous case)
downloader = CurlDownload(protocol_name, server, remote_dir,
http_parse)
if protocol == 4: # DirectFTP
downloader = DirectFTPDownload("ftp", server, '/')
if protocol == 5: # DirectHTTP
downloader = DirectHTTPDownload("http", server, '/')
if protocol == 6: # DirectHTTPS
downloader = DirectHTTPDownload("https", server, '/')
if protocol == 10: # DirectFTPS
downloader = DirectFTPDownload('ftps', server, '/')
if protocol == 7: # Local
downloader = LocalDownload(remote_dir)
if protocol == 8: # RSYNC
downloader = RSYNCDownload(server, remote_dir)
if protocol == 9: # iRods
downloader = IRODSDownload(server, remote_dir)
if downloader is None:
return None
for remote_file in remote_files:
if remote_file['save_as']:
save_as = remote_file['save_as']
# For direct protocols, we only keep base name
if protocol in DIRECT_PROTOCOLS:
tmp_remote = []
for remote_file in remote_files:
tmp_remote.append(remote_file['name'])
remote_files = tmp_remote
if http_method is not None:
downloader.set_method(http_method)
if offline_dir:
downloader.set_offline_dir(offline_dir)
if proxy is not None and proxy:
downloader.set_proxy(proxy, proxy_auth)
if timeout_download is not None and timeout_download:
downloader.set_timeout(timeout_download)
if credentials:
downloader.set_credentials(credentials)
if save_as:
downloader.set_save_as(save_as)
if param:
downloader.set_param(param)
downloader.set_server(server)
# Set the name of the BioMAJ protocol to which we respond.
downloader.set_protocol(protocol_name)
if options is not None:
self.logger.debug("Received options: " + str(options))
downloader.set_options(options)
downloader.logger = self.logger
downloader.set_files_to_download(remote_files)
return downloader
