def _build(self, dataset_type, config):
download_folder = os.path.join(get_pythia_root(), config.data_root_dir, config.data_folder)
file_name = CLEVR_DOWNLOAD_URL.split("/")[-1]
local_filename = os.path.join(download_folder, file_name)
extraction_folder = os.path.join(download_folder, ".".join(file_name.split(".")[:-1]))
self.data_folder = extraction_folder
# Either if the zip file is already present or if there are some
# files inside the folder we don't continue download process
if os.path.exists(local_filename):
self.writer.write("CLEVR dataset is already present. Skipping download.")
return
if os.path.exists(extraction_folder) and \
len(os.listdir(extraction_folder)) != 0:
return
self.writer.write("Downloading the CLEVR dataset now")
download_file(CLEVR_DOWNLOAD_URL, output_dir=download_folder)
self.writer.write("Downloaded. Extracting now. This can take time.")
with zipfile.ZipFile(local_filename, "r") as zip_ref:
zip_ref.extractall(download_folder)
def _try_downloading_necessities(self):
if self.args.model_file is None:
print("Downloading model and configuration")
self.args.model_file = self.MODEL_URL.split("/")[-1]
self.args.config_file = self.CONFIG_URL.split("/")[-1]
download_file(self.MODEL_URL)
download_file(self.CONFIG_URL)
def _download_and_extract(self, key, url, download_folder):
file_type = key.split("_")[0]
os.makedirs(download_folder, exist_ok=True)
local_filename = url.split("/")[-1]
extraction_folder = os.path.join(download_folder,
local_filename.split(".")[0])
local_filename = os.path.join(download_folder, local_filename)
if os.path.exists(local_filename) or \
(os.path.exists(extraction_folder) and len(os.listdir(extraction_folder))) != 0:
self.writer.write(
"{} {} already present. Skipping download.".format(
self.dataset_proper_name, file_type))
return extraction_folder
self.writer.write("Downloading the {} {} now.".format(
self.dataset_proper_name, file_type))
download_file(url, output_dir=download_folder)
self.writer.write(
"Extracting the {} {} now. This may take time".format(
self.dataset_proper_name, file_type))
extract_file(local_filename, output_dir=download_folder)
return extraction_folder
def _download_and_extract(self, key, download_folder):
file_type = key.split("_")[0]
os.makedirs(download_folder, exist_ok=True)
local_filename = VISUAL_GENOME_CONSTS[key].split("/")[-1]
extraction_folder = os.path.join(download_folder,
local_filename.split(".")[0])
local_filename = os.path.join(download_folder, local_filename)
if os.path.exists(local_filename) or \
(os.path.exists(extraction_folder) and len(os.listdir(extraction_folder))) != 0:
self.writer.write(
"Visual Genome {} already present. Skipping download.".format(
file_type))
return extraction_folder
self.writer.write(
"Downloading the Visual Genome {} now.".format(file_type))
download_file(VISUAL_GENOME_CONSTS[key], output_dir=download_folder)
self.writer.write(
"Extracting the Visual Genome {} now. This may take time".format(
file_type))
extract_file(local_filename, output_dir=download_folder)
return extraction_folder