def test_unsupported_protocol(self): urlsList = ['file://path/to/file.txt'] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.FAILURE)
def test_https_downloader_failure(self): urlsList = ['https://i.imgur.com/slsdjfldjfl.jpg'] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.FAILURE)
def test_https_downloader_success(self): urlsList = ['https://i.imgur.com/slmM8rc.jpg'] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.SUCCESS)
def test_parse_ftp_url_login_success(self): url = 'ftp://*****:*****@speedtest.tele2.net:21/path/to/file/512KB.zip' urlInfo = GenericDownloader.parseUrl(url) urlInfo.outputFilenameSuffix = '' expectedUrlInfo = UrlInfo(url) expectedUrlInfo.isValid = True expectedUrlInfo.message = '' expectedUrlInfo.port = 21 expectedUrlInfo.outputFilename = '512KB' expectedUrlInfo.outputFilenameExtension = 'zip' expectedUrlInfo.outputFilenameSuffix = '' expectedUrlInfo.scheme = 'ftp' expectedUrlInfo.netloc = 'demo-user:[email protected]:21' expectedUrlInfo.hostname = 'speedtest.tele2.net' expectedUrlInfo.path = '/path/to/file/512KB.zip' expectedUrlInfo.params = '' expectedUrlInfo.query = '' expectedUrlInfo.fragment = '' expectedUrlInfo.username = '******' expectedUrlInfo.password = '******' expectedUrlInfo.dirName = '/path/to/file' self.assertEqual(urlInfo, expectedUrlInfo)
def test_ftp_anon_downloader_success(self): urlsList = ['ftp://speedtest.tele2.net/512KB.zip'] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.SUCCESS)
def test_ftp_anon_downloader_file_not_found(self): urlsList = ['ftp://speedtest.tele2.net/path/to/file/512KB.zip'] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.FAILURE)
def test_parse_http_url_success(self): url = 'https://i.imgur.com/slmM8rc.jpg' urlInfo = GenericDownloader.parseUrl(url) urlInfo.outputFilenameSuffix = '' expectedUrlInfo = UrlInfo(url) expectedUrlInfo.inputUrl = 'https://i.imgur.com/slmM8rc.jpg' expectedUrlInfo.isValid = True expectedUrlInfo.message = '' expectedUrlInfo.port = 0 expectedUrlInfo.outputFilename = 'slmM8rc' expectedUrlInfo.outputFilenameExtension = 'jpg' expectedUrlInfo.outputFilenameSuffix = '' expectedUrlInfo.scheme = 'https' expectedUrlInfo.netloc = 'i.imgur.com' expectedUrlInfo.hostname = 'i.imgur.com' expectedUrlInfo.path = '/slmM8rc.jpg' expectedUrlInfo.params = '' expectedUrlInfo.query = '' expectedUrlInfo.fragment = '' expectedUrlInfo.username = None expectedUrlInfo.password = None expectedUrlInfo.dirName = '' self.assertEqual(urlInfo, expectedUrlInfo)
def test_nonstandard_input_format_from_input_file(self): downloader = GenericDownloader.fromInputFile( numThreads=1, sourceList='.\\tests\\data\\nonstandard_input_format.in', sourceListDelimiter=',', destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.SUCCESS)
def test_sftp_auth_failure(self): urlsList = [ 'sftp://*****:*****@demo.wftpserver.com:2222/download/manual_en.pdf' ] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.FAILURE)
def test_sftp_success(self): urlsList = [ 'sftp://*****:*****@demo.wftpserver.com:2222/download/manual_en.pdf' ] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.SUCCESS)
def test_nonstandard_input_format_from_list(self): urlsList = [ ' https://i.imgur.com/mINAmnD.gifv ', '', ' https://i.imgur.com/Zd2ybNv.png' ] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.SUCCESS)
def test_https_downloader_warning(self): urlsList = [ 'https://i.imgur.com/slmM8rc.jpg', 'https://i.imgur.com/sldjflsdjfs.jpg' ] downloader = GenericDownloader.fromList(numThreads=1, urlsList=urlsList, destination=self.outputDir) result = downloader.startDownloads() self.assertEqual(result, Status.WARNING)
def test_parse_input_sourcses_success(self): pathToInput = '.\\tests\\data\\nonstandard_input_format.in' urlsList = GenericDownloader.parseInputSources(pathToInput, ',') expectedUrlsList = [ '', ' https://i.imgur.com/Zd2ybNv.png', 'https://i.imgur.com/SnRktzt.gifv', 'https://i.imgur.com/mINAmnD.gifv', 'https://i.imgur.com/slmM8rc.jpg' ] self.assertEqual(urlsList, expectedUrlsList)
def test_clean_urls_lists(self): urlsList = [ ' https://i.imgur.com/mINAmnD.gifv ', '', ' https://i.imgur.com/Zd2ybNv.png', 'https://i.imgur.com/Zd2ybNv.png' ] cleanUrlsList = GenericDownloader.cleanUrlsList(urlsList) expectedUrlList = [ 'https://i.imgur.com/Zd2ybNv.png', 'https://i.imgur.com/mINAmnD.gifv' ] self.assertEqual(cleanUrlsList, expectedUrlList)
def test_parse_ftp_url_anon_success(self): url = 'ftp://speedtest.tele2.net/512KB.zip' urlInfo = GenericDownloader.parseUrl(url) urlInfo.outputFilenameSuffix = '' expectedUrlInfo = UrlInfo(url) expectedUrlInfo.isValid = True expectedUrlInfo.message = '' expectedUrlInfo.port = 0 expectedUrlInfo.outputFilename = '512KB' expectedUrlInfo.outputFilenameExtension = 'zip' expectedUrlInfo.outputFilenameSuffix = '' expectedUrlInfo.scheme = 'ftp' expectedUrlInfo.netloc = 'speedtest.tele2.net' expectedUrlInfo.hostname = 'speedtest.tele2.net' expectedUrlInfo.path = '/512KB.zip' expectedUrlInfo.params = '' expectedUrlInfo.query = '' expectedUrlInfo.fragment = '' expectedUrlInfo.username = None expectedUrlInfo.password = None expectedUrlInfo.dirName = '' self.assertEqual(urlInfo, expectedUrlInfo)
def test_output_filepath_empty(self): with self.assertRaises(ValueError): GenericDownloader.fromInputFile( numThreads=1, sourceList='.\\tests\\data\\https_success.in', destination='')
def test_parse_http_url_invalid(self): url = 'INVALID_URL' urlInfo = GenericDownloader.parseUrl(url) self.assertEqual(urlInfo.isValid, False)
def test_parse_input_sources_not_found(self): pathToInvalidInput = '.\\path\\to\\not_found.in' with self.assertRaises(FileNotFoundError): GenericDownloader.parseInputSources(pathToInvalidInput, ',')
def test_empty_urls_list(self): with self.assertRaises(ValueError): GenericDownloader.fromList(numThreads=1, urlsList=[], destination=self.outputDir)
def test_parse_input_sources_invalid_input_source_parameter(self): with self.assertRaises(ValueError): GenericDownloader.parseInputSources('', ',')
def test_input_filepath_empty(self): with self.assertRaises(ValueError): GenericDownloader.fromInputFile(numThreads=1, sourceList='', destination=self.outputDir)
def test_input_filepath_not_found(self): with self.assertRaises(FileNotFoundError): GenericDownloader.fromInputFile(numThreads=1, sourceList='.\\does_not_exist.in', sourceListDelimiter=',', destination=self.outputDir)
def test_empty_input_file(self): with self.assertRaises(ValueError): GenericDownloader.fromInputFile( numThreads=1, sourceList='.\\tests\\data\\empty.in', destination=self.outputDir)
def main(argv): helpMsg = 'file_downloader.py -s <sourcelist> -d <destination> [-n <numthreads=5> -c <chunksize=8192> -t <timeout=60.0> -r <delimiter=none> -l <logLevel>]' sourceList = '' destination = '' config = configparser.ConfigParser() config.read('./config/file_downloader.ini') defaults = config['DEFAULT'] numThreads = int(defaults['numThreads']) if 'numThreads' in defaults else 5 chunkSize = int(defaults['chunkSize']) if 'chunkSize' in defaults else 8192 timeout = float(defaults['timeout']) if 'timeout' in defaults else 60.0 delimiter = delimiter = defaults[ 'delimiter'] if 'delimiter' in defaults else None logLevel = logLevel = defaults[ 'logLevel'] if 'logLevel' in defaults else 'INFO' try: opts, args = getopt.getopt(argv, "hs:d:n:c:t:r:l:") except: print(helpMsg) sys.exit(2) for opt, arg in opts: if opt == '-h': print(helpMsg) sys.exit() elif opt in ('-s'): sourceList = arg elif opt in ('-d'): destination = arg elif opt in ('-n'): numThreads = int(arg) elif opt in ('-c'): chunkSize = int(arg) elif opt in ('-t'): timeout = float(arg) elif opt in ('-r'): delimiter = arg elif opt in ('-l'): logLevel = arg else: print('Unrecognized argument: {}'.format(opt)) if not sourceList or not destination: print(helpMsg) sys.exit(2) try: configureLogger(logLevel) downloader = GenericDownloader.fromInputFile( sourceList=sourceList, sourceListDelimiter=delimiter, numThreads=numThreads, destination=destination, chunkSize=chunkSize, timeout=timeout) downloader.startDownloads() except (ValueError, OSError) as e: print('An unexpected error occured: {}'.format(str(e))) print('Done!')