def _recurse_on_config(self, config):
if OmegaConf.is_list(
config) and len(config) > 0 and "url" in config[0]:
# Found the urls, let's test them
for item in config:
# First try making the DownloadableFile class to make sure
# everything is fine
download = DownloadableFile(**item)
# Now check the actual header 3 times before failing
for i in range(3):
try:
check_header(download._url,
from_google=download._from_google)
break
except AssertionError:
if i == 2:
raise
else:
# If failed, add a sleep of 5 seconds before retrying
time.sleep(2)
elif OmegaConf.is_dict(config):
# Both version and resources should be present
if "version" in config:
self.assertIn("resources", config)
if "resources" in config:
self.assertIn("version", config)
# Let's continue recursing
for item in config:
self._recurse_on_config(config[item])
def _check_download(self, download: DownloadableFileType):
# Check the actual header 3 times before failing
for i in range(3):
try:
check_header(download._url, from_google=download._from_google)
break
except AssertionError:
if i == 2:
raise
else:
# If failed, add a sleep of 5 seconds before retrying
time.sleep(2)
def _recurse_on_config(self, config):
if OmegaConf.is_list(
config) and len(config) > 0 and "url" in config[0]:
# Found the urls, let's test them
for item in config:
# First try making the DownloadableFile class to make sure
# everything is fine
download = DownloadableFile(**item)
# Now check the actual header
check_header(download._url, from_google=download._from_google)
elif OmegaConf.is_dict(config):
# Both version and resources should be present
if "version" in config:
self.assertIn("resources", config)
if "resources" in config:
self.assertIn("version", config)
# Let's continue recursing
for item in config:
self._recurse_on_config(config[item])