def __init__(self,
download_dir,
extract_dir=None,
manual_dir=None,
manual_dir_instructions=None,
dataset_name=None,
force_download=False,
force_extraction=False,
register_checksums=False):
"""Download manager constructor.
Args:
download_dir: `str`, path to directory where downloads are stored.
extract_dir: `str`, path to directory where artifacts are extracted.
manual_dir: `str`, path to manually downloaded/extracted data directory.
manual_dir_instructions: `str`, human readable instructions on how to
prepare contents of the manual_dir for this dataset.
dataset_name: `str`, name of dataset this instance will be used for. If
provided, downloads will contain which datasets they were used for.
force_download: `bool`, default to False. If True, always [re]download.
force_extraction: `bool`, default to False. If True, always [re]extract.
register_checksums: `bool`, default to False. If True, dl checksums aren't
checked, but stored into file.
"""
self._dataset_name = dataset_name
self._download_dir = os.path.expanduser(download_dir)
self._extract_dir = os.path.expanduser(
extract_dir or os.path.join(download_dir, 'extracted'))
self._manual_dir = manual_dir and os.path.expanduser(manual_dir)
self._manual_dir_instructions = manual_dir_instructions
tf.io.gfile.makedirs(self._download_dir)
tf.io.gfile.makedirs(self._extract_dir)
self._force_download = force_download
self._force_extraction = force_extraction
self._register_checksums = register_checksums
# All known URLs: {url: (size, checksum)}
self._sizes_checksums = checksums.get_all_sizes_checksums()
# To record what is being used: {url: (size, checksum)}
self._recorded_sizes_checksums = {}
# These attributes are lazy-initialized since they must be cleared when this
# object is pickled for Beam. They are then recreated on each worker.
self.__downloader = None
self.__extractor = None
def __init__(self,
download_dir,
extract_dir=None,
manual_dir=None,
dataset_name=None,
force_download=False,
force_extraction=False,
register_checksums=False,
ignore_checksums=False):
"""Download manager constructor.
Args:
download_dir: `str`, path to directory where downloads are stored.
extract_dir: `str`, path to directory where artifacts are extracted.
manual_dir: `str`, path to manually downloaded/extracted data directory.
dataset_name: `str`, name of dataset this instance will be used for. If
provided, downloads will contain which datasets they were used for.
force_download: `bool`, default to False. If True, always [re]download.
force_extraction: `bool`, default to False. If True, always [re]extract.
register_checksums: `bool`, default to False. If True, dl checksums aren't
checked, but stored into file.
ignore_checksums: `bool`, default to False. If True, dl checksums aren't
checked or registered. Overrides `register_checksums`.
"""
self._dataset_name = dataset_name
self._download_dir = os.path.expanduser(download_dir)
self._extract_dir = os.path.expanduser(
extract_dir or os.path.join(download_dir, 'extracted'))
self._manual_dir = manual_dir and os.path.expanduser(manual_dir)
tf.io.gfile.makedirs(self._download_dir)
tf.io.gfile.makedirs(self._extract_dir)
self._force_download = force_download
self._force_extraction = force_extraction
self._extractor = extractor.get_extractor()
self._downloader = downloader.get_downloader()
self._register_checksums = register_checksums
# All known URLs: {url: (size, checksum)}
self._sizes_checksums = checksums.get_all_sizes_checksums()
# To record what is being used: {url: (size, checksum)}
self._recorded_sizes_checksums = {}
self._ignore_checksums = ignore_checksums