def test_http_list_dateregexp(self):
#self.http_parse.file_date_format = "%%d-%%b-%%Y %%H:%%M"
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
self.assertTrue(len(file_list) == 1)
def test_download(self):
ftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^readme.txt$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download_skip_check_uncompress(self):
# This test is similar to test_download but we skip test of zip file.
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
ftpd.set_options(dict(skip_check_uncompress=True))
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
def test_http_download_in_subdir(self):
self.http_parse.file_date_format = "%%Y-%%m-%%d %%H:%%M"
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^dists/README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_ftps_list_no_ssl(self):
# This server is misconfigured hence we disable all SSL verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo-user:demo-user"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
def test_download_in_subdir(self):
ftpd = CurlDownload('ftp', 'ftp.fr.debian.org', '/debian/')
(file_list, dir_list) = ftpd.list()
try:
ftpd.match([r'^doc/mailing-lists.txt$'], file_list, dir_list)
except Exception as e:
print("Error: " + str(e))
self.skipTest("Skipping test due to remote server error")
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download_no_ssl(self):
# This server is misconfigured hence we disable all SSL verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo-user:demo-user"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
ftpd.set_options(dict(ssl_verifyhost="False", ssl_verifypeer="False"))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download_ftp_method(self):
"""
Test setting ftp_method (it probably doesn't change anything here but we
test that there is no obvious mistake in the code).
"""
ftpd = CurlDownload("ftp", "test.rebex.net", "/")
ftpd.set_options(dict(ftp_method="nocwd"))
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.match(["^readme.txt$"], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_download(self):
self.utils = UtilsForTest()
self.http_parse = HTTPParse(
"<a[\s]+href=\"([\w\-\.]+\">[\w\-\.]+.tar.gz)<\/a>[\s]+([0-9]{2}-[A-Za-z]{3}-[0-9]{4}[\s][0-9]{2}:[0-9]{2})[\s]+([0-9]+[A-Za-z])",
"<a[\s]+href=\"[\w\-\.]+\">([\w\-\.]+.tar.gz)<\/a>[\s]+([0-9]{2}-[A-Za-z]{3}-[0-9]{4}[\s][0-9]{2}:[0-9]{2})[\s]+([0-9]+[A-Za-z])",
1, 2, 1, 2, None, 3)
self.http_parse.file_date_format = "%%d-%%b-%%Y %%H:%%M"
httpd = CurlDownload('https', 'mirrors.edge.kernel.org',
'/pub/software/scm/git/debian/', self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^git-core-0.99.6.tar.gz$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_download(self):
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^1.*KB\.zip$'], file_list, dir_list)
# This tests fails because the zip file is fake. We intercept the failure
# and continue.
# See test_download_skip_check_uncompress
try:
ftpd.download(self.utils.data_dir)
except Exception:
self.assertTrue(1 == 1)
else:
# In case it works, this is the real assertion
self.assertTrue(len(ftpd.files_to_download) == 2)
ftpd.close()
def test_http_download_no_date(self):
self.http_parse = HTTPParse(
self.config.get('http.parse.dir.line'),
self.config.get('http.parse.file.line'),
int(self.config.get('http.group.dir.name')),
int(self.config.get('http.group.dir.date')),
int(self.config.get('http.group.file.name')), -1,
self.config.get('http.group.file.date_format', None),
int(self.config.get('http.group.file.size')))
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.match([r'^README$'], file_list, dir_list)
httpd.download(self.utils.data_dir)
httpd.close()
self.assertTrue(len(httpd.files_to_download) == 1)
def test_download_ssl_certficate(self):
# This server is misconfigured but we use its certificate
# The hostname is wrong so we disable host verification
SERVER = "demo.wftpserver.com"
DIRECTORY = "/download/"
CREDENTIALS = "demo-user:demo-user"
ftpd = CurlDownload(self.PROTOCOL, SERVER, DIRECTORY)
curdir = os.path.dirname(os.path.realpath(__file__))
cert_file = os.path.join(curdir, "caert.demo.wftpserver.com.pem")
ftpd.set_options(
dict(ssl_verifyhost="False", ssl_server_cert=cert_file))
ftpd.set_credentials(CREDENTIALS)
(file_list, dir_list) = ftpd.list()
ftpd.match([r'^manual_en.pdf$'], file_list, dir_list)
ftpd.download(self.utils.data_dir)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 1)
def test_ftps_list(self):
ftpd = CurlDownload(self.PROTOCOL, "test.rebex.net", "/")
ftpd.set_credentials("demo:password")
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) == 1)
def test_download_or_copy(self):
ftpd = CurlDownload('ftp', 'ftp.fr.debian.org', '/debian/')
ftpd.files_to_download = [{
'name': '/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test2',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test11',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}]
available_files = [{
'name': '/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test12',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test3',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}, {
'name': '/test/test1',
'year': '2013',
'month': '11',
'day': '10',
'size': 20
}, {
'name': '/test/test11',
'year': '2013',
'month': '11',
'day': '10',
'size': 10
}]
ftpd.download_or_copy(available_files, '/biomaj', False)
ftpd.close()
self.assertTrue(len(ftpd.files_to_download) == 2)
self.assertTrue(len(ftpd.files_to_copy) == 2)
def test_ftp_list(self):
ftpd = CurlDownload('ftp', 'speedtest.tele2.net', '/')
(file_list, dir_list) = ftpd.list()
ftpd.close()
self.assertTrue(len(file_list) > 1)
def test_http_list(self):
httpd = CurlDownload('http', 'ftp2.fr.debian.org', '/debian/dists/',
self.http_parse)
(file_list, dir_list) = httpd.list()
httpd.close()
self.assertTrue(len(file_list) == 1)