def test_download(self):
"""Test download method"""
filename = 'download_test.txt'
# standard download
test_url = urljoin(self.wdir, filename)
scraper = mozdownload.DirectScraper(url=test_url,
destination=self.temp_dir,
logger=self.logger)
scraper.download()
self.assertTrue(os.path.isfile(os.path.join(self.temp_dir, filename)))
# Compare original and downloaded file via md5 hash
md5_original = create_md5(
os.path.join(mhttpd.HERE, mhttpd.WDIR, filename))
md5_downloaded = create_md5(os.path.join(self.temp_dir, filename))
self.assertEqual(md5_original, md5_downloaded)
# RequestException
test_url1 = urljoin(self.wdir, 'does_not_exist.html')
scraper1 = mozdownload.DirectScraper(url=test_url1,
destination=self.temp_dir,
logger=self.logger)
self.assertRaises(requests.exceptions.RequestException,
scraper1.download)
# Covering retry_attempts
test_url2 = urljoin(self.wdir, 'does_not_exist.html')
scraper2 = mozdownload.DirectScraper(url=test_url2,
destination=self.temp_dir,
retry_attempts=3,
retry_delay=1.0,
logger=self.logger)
self.assertRaises(requests.exceptions.RequestException,
scraper2.download)
def test_destination_multiple_dir(httpd, tmpdir):
"""ensure that multiple non existing directories are created"""
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
destination = os.path.join(str(tmpdir), 'tmp1', 'tmp2', filename)
scraper = mozdownload.DirectScraper(url=test_url, destination=destination)
assert scraper.destination == destination
def test_retry_attempts(httpd, tmpdir):
test_url = urljoin(httpd.get_url(), 'does_not_exist.html')
scraper = mozdownload.DirectScraper(url=test_url,
destination=str(tmpdir),
retry_attempts=3,
retry_delay=0.1)
with pytest.raises(errors.NotFoundError):
scraper.download()
def test_retry_attempts(httpd, tmpdir):
test_url = urljoin(httpd.get_url(), 'does_not_exist.html')
scraper = mozdownload.DirectScraper(url=test_url,
destination=str(tmpdir),
retry_attempts=3,
retry_delay=0.1)
with pytest.raises(requests.exceptions.RequestException):
scraper.download()
def test_compare_download(httpd, tmpdir):
"""Compare original and downloaded file via md5 hash"""
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
scraper = mozdownload.DirectScraper(url=test_url, destination=str(tmpdir))
scraper.download()
md5_original = create_md5(os.path.join(httpd.router.doc_root, filename))
md5_downloaded = create_md5(os.path.join(str(tmpdir), filename))
assert md5_original == md5_downloaded
def test_valid_authentication(tmpdir):
username = '******'
password = '******'
basic_auth_url = 'http://mozqa.com/data/mozqa.com/http_auth/basic/'
scraper = mozdownload.DirectScraper(destination=str(tmpdir),
url=basic_auth_url,
username=username,
password=password)
scraper.download()
assert os.path.isfile(os.path.join(str(tmpdir), 'mozqa.com'))
def test_valid_authentication(httpd, tmpdir):
username = '******'
password = '******'
basic_auth_url = urljoin(httpd.get_url(), 'basic_auth')
scraper = mozdownload.DirectScraper(destination=str(tmpdir),
url=basic_auth_url,
username=username,
password=password)
scraper.download()
assert os.path.isfile(os.path.join(str(tmpdir), 'basic_auth'))
def test_authentication(self):
"""testing with basic authentication"""
username = '******'
password = '******'
basic_auth_url = 'http://mozqa.com/data/mozqa.com/http_auth/basic/'
# test with invalid authentication
scraper = mozdownload.DirectScraper(destination=self.temp_dir,
url=basic_auth_url,
logger=self.logger)
self.assertRaises(requests.exceptions.HTTPError, scraper.download)
# testing with valid authentication
scraper = mozdownload.DirectScraper(destination=self.temp_dir,
url=basic_auth_url,
logger=self.logger,
username=username,
password=password)
scraper.download()
self.assertTrue(
os.path.isfile(os.path.join(self.temp_dir, 'mozqa.com')))
def test_optional_authentication(self):
"""testing with optional basic authentication"""
optional_auth_url = 'https://webqa-ci.mozilla.com/'
# requires optional authentication with no data specified
scraper = mozdownload.DirectScraper(destination=self.temp_dir,
url=optional_auth_url,
logger=self.logger)
scraper.download()
self.assertTrue(
os.path.isfile(os.path.join(self.temp_dir,
'webqa-ci.mozilla.com')))
def test_destination(self):
"""Test for various destination scenarios"""
filename = 'download_test.txt'
test_url = urljoin(self.wdir, filename)
# destination is directory
scraper = mozdownload.DirectScraper(url=test_url,
destination=self.temp_dir,
logger=self.logger)
self.assertEqual(scraper.filename,
os.path.join(self.temp_dir, filename))
# destination has directory path with filename
destination = os.path.join(self.temp_dir, filename)
scraper = mozdownload.DirectScraper(url=test_url,
destination=destination,
logger=self.logger)
self.assertEqual(scraper.filename, destination)
# destination only has filename
scraper = mozdownload.DirectScraper(url=test_url,
destination=filename,
logger=self.logger)
self.assertEqual(scraper.filename, os.path.abspath(filename))
# destination directory does not exist
destination = os.path.join(self.temp_dir, 'temp_folder', filename)
scraper = mozdownload.DirectScraper(url=test_url,
destination=destination,
logger=self.logger)
self.assertEqual(scraper.destination, destination)
# ensure that multiple non existing directories are created
destination = os.path.join(self.temp_dir, 'tmp1', 'tmp2', filename)
scraper = mozdownload.DirectScraper(url=test_url,
destination=destination,
logger=self.logger)
self.assertEqual(scraper.destination, destination)
def test_invalid_authentication(httpd, tmpdir):
basic_auth_url = urljoin(httpd.get_url(), 'basic_auth')
scraper = mozdownload.DirectScraper(destination=str(tmpdir),
url=basic_auth_url)
with pytest.raises(requests.exceptions.HTTPError):
scraper.download()
def test_url_not_found(httpd, tmpdir):
test_url = urljoin(httpd.get_url(), 'does_not_exist.html')
scraper = mozdownload.DirectScraper(url=test_url, destination=str(tmpdir))
with pytest.raises(errors.NotFoundError):
scraper.download()
def test_standard_download(httpd, tmpdir):
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
scraper = mozdownload.DirectScraper(url=test_url, destination=str(tmpdir))
scraper.download()
assert os.path.isfile(os.path.join(str(tmpdir), filename))
def test_destination_as_filename_only(httpd):
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
scraper = mozdownload.DirectScraper(url=test_url, destination=filename)
assert scraper.filename == os.path.abspath(filename)
def test_invalid_authentication(tmpdir):
basic_auth_url = 'http://mozqa.com/data/mozqa.com/http_auth/basic/'
scraper = mozdownload.DirectScraper(destination=str(tmpdir),
url=basic_auth_url)
with pytest.raises(requests.exceptions.HTTPError):
scraper.download()
def test_url_not_found(httpd, tmpdir):
test_url = urljoin(httpd.get_url(), 'does_not_exist.html')
scraper = mozdownload.DirectScraper(url=test_url, destination=str(tmpdir))
with pytest.raises(requests.exceptions.RequestException):
scraper.download()
def test_destination_as_path_with_filename(httpd, tmpdir):
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
destination = os.path.join(str(tmpdir), filename)
scraper = mozdownload.DirectScraper(url=test_url, destination=destination)
assert scraper.filename == destination
def test_destination_does_not_exist(httpd, tmpdir):
filename = 'download_test.txt'
test_url = urljoin(httpd.get_url(), filename)
destination = os.path.join(str(tmpdir), 'temp_folder', filename)
scraper = mozdownload.DirectScraper(url=test_url, destination=destination)
assert scraper.destination == destination
