def test_dlfile(self, m):
"""Test for `dl_file` standard beahavior."""
# Test for an available text file
url = self.test_url + '/test.txt'
file_content = 'resp'
m.get(url, text=file_content)
out_file = dl_file(url, self.tempdir)
with open(out_file) as f:
self.assertEqual(file_content, f.read())
# Test for an available text file with parameters in the url
url = self.test_url + '/test.txt?id=1&out=45'
file_content = 'resp'
m.get(url, text=file_content)
out_file = dl_file(url, self.tempdir)
# Check if parameters have been correctly removed
self.assertEqual('test.txt', fm.split(out_file)[1])
with open(out_file) as f:
self.assertEqual(file_content, f.read())
# Test for an unavailable text file
url = self.test_url + '/test2.txt'
m.get(url, text='Not Found', status_code=404)
with self.assertRaises(RuntimeError) as e:
dl_file(url, self.tempdir)
self.assertTrue(
'File is not available (error code 404)' in str(e.exception))
def test_dlfile_headers(self, m):
"""Test for `dl_file` beahavior with additional headers."""
# Test for an available text file (with 'content-length' header)
url = self.test_url + '/test.txt'
file_content = 'resp'
m.get(url, text=file_content, request_headers={'TOKEN': 'test'})
out_file = dl_file(url, self.tempdir, headers={'TOKEN': 'test'})
with open(out_file) as f:
self.assertEqual(file_content, f.read())
# Additional headers should be ignored
out_file = dl_file(url,
self.tempdir,
headers={
'TOKEN': 'test',
'HEADER2': '2'
})
with open(out_file) as f:
self.assertEqual(file_content, f.read())
# download should fail if no header or wrong header is defined
with self.assertRaises(Exception):
dl_file(url, self.tempdir)
with self.assertRaises(Exception):
dl_file(url, self.tempdir, headers={'TOKEN': '2'})
with self.assertRaises(Exception):
dl_file(url, self.tempdir, headers={'HEADER2': 'test'})
def _test_dlfile(cont_disp, fn='test.txt'):
"""
Parameters
----------
cont_disp : str
Content of the 'Content-Disposition' header
fn : str, optional
Expected filename. Default 'text.txt'.
"""
url = self.test_url + '/test.txt'
file_content = 'resp'
m.get(url,
text=file_content,
headers={
'Content-Length': '4',
'Content-Disposition': cont_disp
})
out_file = dl_file(url, self.tempdir)
self.assertEqual(fn, fm.split(out_file)[1])
with open(out_file) as f:
self.assertEqual(file_content, f.read())
def _get_data(self, file_url, dl_folder):
"""Download input datafile, unzip and store in output_path.
Parameters
----------
file_url : str
URL of the file to download.
dl_folder : str
Path to the folder where to store the downloaded file.
"""
f_dl = fm.join(dl_folder, 'iCubWorld28_128x128.zip?dl=1')
if not fm.file_exist(f_dl) or md5(f_dl) != ICUBWORLD28_MD5:
# Generate the full path to the downloaded file
f_dl = dl_file(file_url, dl_folder, md5_digest=ICUBWORLD28_MD5)
self.logger.info("Extracting files...")
# Extract the content of downloaded file
zipfile.ZipFile(f_dl, 'r').extractall(dl_folder)
# Remove downloaded file
fm.remove_file(f_dl)
# iCubWorld28 zip file contains a macosx private folder, clean it up
if fm.folder_exist(fm.join(ICUBWORLD28_PATH, '__MACOSX')):
fm.remove_folder(fm.join(ICUBWORLD28_PATH, '__MACOSX'), force=True)
# iCubWorld28 zip file contains a macosx private files, clean it up
for dirpath, dirnames, filenames in os.walk(ICUBWORLD28_PATH):
for file in filenames:
if fnmatch(file, '.DS_Store'):
fm.remove_file(fm.join(dirpath, file))
# Now move all data to an upper folder if needed
if not fm.folder_exist(self._train_path) \
or not fm.folder_exist(self._test_path):
sub_d = fm.join(dl_folder, fm.listdir(dl_folder)[0])
for e in fm.listdir(sub_d):
e_full = fm.join(sub_d, e) # Full path to current element
try: # Call copy_file or copy_folder when applicable
if fm.file_exist(e_full) is True:
fm.copy_file(e_full, dl_folder)
elif fm.folder_exist(e_full) is True:
fm.copy_folder(e_full, fm.join(dl_folder, e))
except:
pass
# Check that the main dataset file is now in the correct folder
if not fm.folder_exist(self._train_path) \
or not fm.folder_exist(self._test_path):
raise RuntimeError("dataset main file not available!")
# The subdirectory can now be removed
fm.remove_folder(sub_d, force=True)
def test_dlfile_content_length(self, m):
"""Test for `dl_file` beahavior with 'content-length' header."""
# Test for an available text file (with 'content-length' header)
url = self.test_url + '/test.txt'
file_content = 'resp'
m.get(url, text=file_content, headers={'Content-Length': '4'})
out_file = dl_file(url, self.tempdir)
with open(out_file) as f:
self.assertEqual(file_content, f.read())
def _get_data(self, file_url, dl_folder, output_path):
"""Download input datafile, unzip and store in output_path.
Parameters
----------
file_url : str
URL of the file to download.
dl_folder : str
Path to the folder where to store the downloaded file.
output_path : str
Full path of output file.
"""
# Download file and unpack
fh = dl_file(file_url, dl_folder)
with gzip.open(fh, 'rb') as infile:
with open(output_path, 'wb') as outfile:
for line in infile:
outfile.write(line)
# Remove download zipped file
fm.remove_file(fh)
def _get_data(self, file_url, dl_folder, extract_only=False):
"""Download input datafile, unzip and store in output_path.
Parameters
----------
file_url : str
URL of the file to download.
dl_folder : str
Path to the folder where to store the downloaded file.
extract_only : bool, optional
If True, only extract data from the datafile. Default False.
"""
# Generate the full path to the downloaded file
f = fm.join(dl_folder, self.data_url.split('/')[-1])
if extract_only is False:
f_dl = dl_file(file_url, dl_folder, md5_digest=self.data_md5)
if f != f_dl:
raise ValueError("Unexpected filename {:}".format(f_dl))
tarfile.open(name=f, mode='r:gz').extractall(dl_folder)
