def download(self, is_force: bool = False):
self._new_folder_list.append(self._download_folder)
os.makedirs(self._download_folder, exist_ok=True)
self._new_file_list.append(self._download_file)
if (not is_force) and os.path.exists(self._download_file):
logger.info_green("File already exists, skipping download.")
else:
logger.info_green(
f"Downloading data from {self._source} to {self._download_file}."
)
try:
download_file(source=self._source,
destination=self._download_file)
except Exception as e:
logger.warning_yellow(
f"Failed to download from {self._source} to {self._download_file}."
)
raise CommandError("generate", f"Download error: {e}.")
# Download text with all urls.
if os.path.exists(self._download_file):
# Download vm_table and cpu_readings
self._aria2p_download(is_force=is_force)
else:
logger.warning(
f"Not found downloaded source file: {self._download_file}.")
def download(self, is_force: bool, fall_back: callable = None):
"""Download the original data file.
Args:
is_force(bool): If forced re-download the data file.
fall_back(callable): (optional) Fallback function to execute when download failed.
"""
self._new_folder_list.append(self._download_folder)
os.makedirs(self._download_folder, exist_ok=True)
self._new_file_list.append(self._download_file)
if (not is_force) and os.path.exists(self._download_file):
logger.info_green("File already exists, skipping download.")
else:
logger.info_green(
f"Downloading data from {self._source} to {self._download_file}."
)
try:
download_file(source=self._source,
destination=self._download_file)
except Exception as e:
logger.warning_yellow(
f"Failed to download from {self._source} to {self._download_file}."
)
if fall_back is not None:
logger.warning_yellow(
f"Calling fall_back function: {fall_back}.")
fall_back()
else:
raise CommandError("generate", f"Download error: {e}.")
def _download_processed_data(self):
"""Build processed data."""
data_root = StaticParameter.data_root
build_folder = os.path.join(data_root, self._scenario_name, ".build",
self._topology)
source = self._config.PROCESSED_DATA_URL
download_file_name = source.split('/')[-1]
download_file_path = os.path.join(build_folder, download_file_name)
# Download file from the Azure blob storage.
if not os.path.exists(download_file_path):
logger.info_green(
f"Downloading data from {source} to {download_file_path}.")
download_file(source=source, destination=download_file_path)
else:
logger.info_green("File already exists, skipping download.")
logger.info_green(f"Unzip {download_file_path} to {build_folder}")
# Unzip files.
tar = tarfile.open(download_file_path, "r:gz")
tar.extractall(path=build_folder)
tar.close()
# Move to the correct path.
unzip_file = os.path.join(build_folder, "build")
file_names = os.listdir(unzip_file)
for file_name in file_names:
shutil.move(os.path.join(unzip_file, file_name), build_folder)
os.rmdir(unzip_file)
def download(self, is_force: bool):
"""download the original data file"""
self._new_folder_list.append(self._download_folder)
os.makedirs(self._download_folder, exist_ok=True)
self._new_file_list.append(self._download_file)
if (not is_force) and os.path.exists(self._download_file):
logger.info_green("File already exists, skipping download.")
else:
logger.info_green(f"Downloading data from {self._source} to {self._download_file}")
download_file(source=self._source, destination=self._download_file)
def download(self, is_force: bool = False):
"""download the zip file"""
super().download(is_force)
self._new_file_list.append(self._station_info_file)
if (not is_force) and os.path.exists(self._station_info_file):
logger.info_green("File already exists, skipping download.")
else:
logger.info_green(
f"Downloading trip data from {self._station_info} to {self._station_info_file}"
)
download_file(source=self._station_info,
destination=self._station_info_file)