def test_unsupported_protocol(self):
urlsList = ['file://path/to/file.txt']
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.FAILURE)
def test_https_downloader_failure(self):
urlsList = ['https://i.imgur.com/slsdjfldjfl.jpg']
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.FAILURE)
def test_https_downloader_success(self):
urlsList = ['https://i.imgur.com/slmM8rc.jpg']
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.SUCCESS)
def test_parse_ftp_url_login_success(self):
url = 'ftp://*****:*****@speedtest.tele2.net:21/path/to/file/512KB.zip'
urlInfo = GenericDownloader.parseUrl(url)
urlInfo.outputFilenameSuffix = ''
expectedUrlInfo = UrlInfo(url)
expectedUrlInfo.isValid = True
expectedUrlInfo.message = ''
expectedUrlInfo.port = 21
expectedUrlInfo.outputFilename = '512KB'
expectedUrlInfo.outputFilenameExtension = 'zip'
expectedUrlInfo.outputFilenameSuffix = ''
expectedUrlInfo.scheme = 'ftp'
expectedUrlInfo.netloc = 'demo-user:[email protected]:21'
expectedUrlInfo.hostname = 'speedtest.tele2.net'
expectedUrlInfo.path = '/path/to/file/512KB.zip'
expectedUrlInfo.params = ''
expectedUrlInfo.query = ''
expectedUrlInfo.fragment = ''
expectedUrlInfo.username = '******'
expectedUrlInfo.password = '******'
expectedUrlInfo.dirName = '/path/to/file'
self.assertEqual(urlInfo, expectedUrlInfo)
def test_ftp_anon_downloader_success(self):
urlsList = ['ftp://speedtest.tele2.net/512KB.zip']
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.SUCCESS)
def test_ftp_anon_downloader_file_not_found(self):
urlsList = ['ftp://speedtest.tele2.net/path/to/file/512KB.zip']
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.FAILURE)
def test_parse_http_url_success(self):
url = 'https://i.imgur.com/slmM8rc.jpg'
urlInfo = GenericDownloader.parseUrl(url)
urlInfo.outputFilenameSuffix = ''
expectedUrlInfo = UrlInfo(url)
expectedUrlInfo.inputUrl = 'https://i.imgur.com/slmM8rc.jpg'
expectedUrlInfo.isValid = True
expectedUrlInfo.message = ''
expectedUrlInfo.port = 0
expectedUrlInfo.outputFilename = 'slmM8rc'
expectedUrlInfo.outputFilenameExtension = 'jpg'
expectedUrlInfo.outputFilenameSuffix = ''
expectedUrlInfo.scheme = 'https'
expectedUrlInfo.netloc = 'i.imgur.com'
expectedUrlInfo.hostname = 'i.imgur.com'
expectedUrlInfo.path = '/slmM8rc.jpg'
expectedUrlInfo.params = ''
expectedUrlInfo.query = ''
expectedUrlInfo.fragment = ''
expectedUrlInfo.username = None
expectedUrlInfo.password = None
expectedUrlInfo.dirName = ''
self.assertEqual(urlInfo, expectedUrlInfo)
def test_nonstandard_input_format_from_input_file(self):
downloader = GenericDownloader.fromInputFile(
numThreads=1,
sourceList='.\\tests\\data\\nonstandard_input_format.in',
sourceListDelimiter=',',
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.SUCCESS)
def test_sftp_auth_failure(self):
urlsList = [
'sftp://*****:*****@demo.wftpserver.com:2222/download/manual_en.pdf'
]
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.FAILURE)
def test_sftp_success(self):
urlsList = [
'sftp://*****:*****@demo.wftpserver.com:2222/download/manual_en.pdf'
]
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.SUCCESS)
def test_nonstandard_input_format_from_list(self):
urlsList = [
' https://i.imgur.com/mINAmnD.gifv ', '',
' https://i.imgur.com/Zd2ybNv.png'
]
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.SUCCESS)
def test_https_downloader_warning(self):
urlsList = [
'https://i.imgur.com/slmM8rc.jpg',
'https://i.imgur.com/sldjflsdjfs.jpg'
]
downloader = GenericDownloader.fromList(numThreads=1,
urlsList=urlsList,
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.WARNING)
def test_parse_input_sourcses_success(self):
pathToInput = '.\\tests\\data\\nonstandard_input_format.in'
urlsList = GenericDownloader.parseInputSources(pathToInput, ',')
expectedUrlsList = [
'', ' https://i.imgur.com/Zd2ybNv.png',
'https://i.imgur.com/SnRktzt.gifv',
'https://i.imgur.com/mINAmnD.gifv',
'https://i.imgur.com/slmM8rc.jpg'
]
self.assertEqual(urlsList, expectedUrlsList)
def test_clean_urls_lists(self):
urlsList = [
' https://i.imgur.com/mINAmnD.gifv ', '',
' https://i.imgur.com/Zd2ybNv.png',
'https://i.imgur.com/Zd2ybNv.png'
]
cleanUrlsList = GenericDownloader.cleanUrlsList(urlsList)
expectedUrlList = [
'https://i.imgur.com/Zd2ybNv.png',
'https://i.imgur.com/mINAmnD.gifv'
]
self.assertEqual(cleanUrlsList, expectedUrlList)
def test_parse_ftp_url_anon_success(self):
url = 'ftp://speedtest.tele2.net/512KB.zip'
urlInfo = GenericDownloader.parseUrl(url)
urlInfo.outputFilenameSuffix = ''
expectedUrlInfo = UrlInfo(url)
expectedUrlInfo.isValid = True
expectedUrlInfo.message = ''
expectedUrlInfo.port = 0
expectedUrlInfo.outputFilename = '512KB'
expectedUrlInfo.outputFilenameExtension = 'zip'
expectedUrlInfo.outputFilenameSuffix = ''
expectedUrlInfo.scheme = 'ftp'
expectedUrlInfo.netloc = 'speedtest.tele2.net'
expectedUrlInfo.hostname = 'speedtest.tele2.net'
expectedUrlInfo.path = '/512KB.zip'
expectedUrlInfo.params = ''
expectedUrlInfo.query = ''
expectedUrlInfo.fragment = ''
expectedUrlInfo.username = None
expectedUrlInfo.password = None
expectedUrlInfo.dirName = ''
self.assertEqual(urlInfo, expectedUrlInfo)
def test_output_filepath_empty(self):
with self.assertRaises(ValueError):
GenericDownloader.fromInputFile(
numThreads=1,
sourceList='.\\tests\\data\\https_success.in',
destination='')
def test_parse_http_url_invalid(self):
url = 'INVALID_URL'
urlInfo = GenericDownloader.parseUrl(url)
self.assertEqual(urlInfo.isValid, False)
def test_parse_input_sources_not_found(self):
pathToInvalidInput = '.\\path\\to\\not_found.in'
with self.assertRaises(FileNotFoundError):
GenericDownloader.parseInputSources(pathToInvalidInput, ',')
def test_empty_urls_list(self):
with self.assertRaises(ValueError):
GenericDownloader.fromList(numThreads=1,
urlsList=[],
destination=self.outputDir)
def test_parse_input_sources_invalid_input_source_parameter(self):
with self.assertRaises(ValueError):
GenericDownloader.parseInputSources('', ',')
def test_input_filepath_empty(self):
with self.assertRaises(ValueError):
GenericDownloader.fromInputFile(numThreads=1,
sourceList='',
destination=self.outputDir)
def test_input_filepath_not_found(self):
with self.assertRaises(FileNotFoundError):
GenericDownloader.fromInputFile(numThreads=1,
sourceList='.\\does_not_exist.in',
sourceListDelimiter=',',
destination=self.outputDir)
def test_empty_input_file(self):
with self.assertRaises(ValueError):
GenericDownloader.fromInputFile(
numThreads=1,
sourceList='.\\tests\\data\\empty.in',
destination=self.outputDir)
def main(argv):
helpMsg = 'file_downloader.py -s <sourcelist> -d <destination> [-n <numthreads=5> -c <chunksize=8192> -t <timeout=60.0> -r <delimiter=none> -l <logLevel>]'
sourceList = ''
destination = ''
config = configparser.ConfigParser()
config.read('./config/file_downloader.ini')
defaults = config['DEFAULT']
numThreads = int(defaults['numThreads']) if 'numThreads' in defaults else 5
chunkSize = int(defaults['chunkSize']) if 'chunkSize' in defaults else 8192
timeout = float(defaults['timeout']) if 'timeout' in defaults else 60.0
delimiter = delimiter = defaults[
'delimiter'] if 'delimiter' in defaults else None
logLevel = logLevel = defaults[
'logLevel'] if 'logLevel' in defaults else 'INFO'
try:
opts, args = getopt.getopt(argv, "hs:d:n:c:t:r:l:")
except:
print(helpMsg)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(helpMsg)
sys.exit()
elif opt in ('-s'):
sourceList = arg
elif opt in ('-d'):
destination = arg
elif opt in ('-n'):
numThreads = int(arg)
elif opt in ('-c'):
chunkSize = int(arg)
elif opt in ('-t'):
timeout = float(arg)
elif opt in ('-r'):
delimiter = arg
elif opt in ('-l'):
logLevel = arg
else:
print('Unrecognized argument: {}'.format(opt))
if not sourceList or not destination:
print(helpMsg)
sys.exit(2)
try:
configureLogger(logLevel)
downloader = GenericDownloader.fromInputFile(
sourceList=sourceList,
sourceListDelimiter=delimiter,
numThreads=numThreads,
destination=destination,
chunkSize=chunkSize,
timeout=timeout)
downloader.startDownloads()
except (ValueError, OSError) as e:
print('An unexpected error occured: {}'.format(str(e)))
print('Done!')
