def test_nonstandard_input_format_from_input_file(self):
downloader = GenericDownloader.fromInputFile(
numThreads=1,
sourceList='.\\tests\\data\\nonstandard_input_format.in',
sourceListDelimiter=',',
destination=self.outputDir)
result = downloader.startDownloads()
self.assertEqual(result, Status.SUCCESS)
def test_empty_input_file(self):
with self.assertRaises(ValueError):
GenericDownloader.fromInputFile(
numThreads=1,
sourceList='.\\tests\\data\\empty.in',
destination=self.outputDir)
def test_output_filepath_empty(self):
with self.assertRaises(ValueError):
GenericDownloader.fromInputFile(
numThreads=1,
sourceList='.\\tests\\data\\https_success.in',
destination='')
def test_input_filepath_empty(self):
with self.assertRaises(ValueError):
GenericDownloader.fromInputFile(numThreads=1,
sourceList='',
destination=self.outputDir)
def test_input_filepath_not_found(self):
with self.assertRaises(FileNotFoundError):
GenericDownloader.fromInputFile(numThreads=1,
sourceList='.\\does_not_exist.in',
sourceListDelimiter=',',
destination=self.outputDir)
def main(argv):
helpMsg = 'file_downloader.py -s <sourcelist> -d <destination> [-n <numthreads=5> -c <chunksize=8192> -t <timeout=60.0> -r <delimiter=none> -l <logLevel>]'
sourceList = ''
destination = ''
config = configparser.ConfigParser()
config.read('./config/file_downloader.ini')
defaults = config['DEFAULT']
numThreads = int(defaults['numThreads']) if 'numThreads' in defaults else 5
chunkSize = int(defaults['chunkSize']) if 'chunkSize' in defaults else 8192
timeout = float(defaults['timeout']) if 'timeout' in defaults else 60.0
delimiter = delimiter = defaults[
'delimiter'] if 'delimiter' in defaults else None
logLevel = logLevel = defaults[
'logLevel'] if 'logLevel' in defaults else 'INFO'
try:
opts, args = getopt.getopt(argv, "hs:d:n:c:t:r:l:")
except:
print(helpMsg)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(helpMsg)
sys.exit()
elif opt in ('-s'):
sourceList = arg
elif opt in ('-d'):
destination = arg
elif opt in ('-n'):
numThreads = int(arg)
elif opt in ('-c'):
chunkSize = int(arg)
elif opt in ('-t'):
timeout = float(arg)
elif opt in ('-r'):
delimiter = arg
elif opt in ('-l'):
logLevel = arg
else:
print('Unrecognized argument: {}'.format(opt))
if not sourceList or not destination:
print(helpMsg)
sys.exit(2)
try:
configureLogger(logLevel)
downloader = GenericDownloader.fromInputFile(
sourceList=sourceList,
sourceListDelimiter=delimiter,
numThreads=numThreads,
destination=destination,
chunkSize=chunkSize,
timeout=timeout)
downloader.startDownloads()
except (ValueError, OSError) as e:
print('An unexpected error occured: {}'.format(str(e)))
print('Done!')
