def convert(meta: str, file: list, output: str, start_timestamp: int = None, **kwargs):
meta_file = meta
csv_files = file
output_file = output
if not os.path.exists(meta_file):
raise CommandError("convert", "meta file not exist.\n")
if not all([os.path.exists(f) for f in csv_files]):
raise CommandError("convert", "some source file not exist.\n")
converter = BinaryConverter(output_file, meta_file, start_timestamp)
for csv_file in csv_files:
converter.add_csv(csv_file)
def _build_temp_data(self):
"""Build temporary data for predefined environment."""
logger.warning_yellow(
f"Binary data files for scenario: citi_bike topology: {self._topology} not found."
)
citi_bike_process = CitiBikeProcess(is_temp=True)
if self._topology in citi_bike_process.topologies:
pid = str(os.getpid())
logger.warning_yellow(
f"Generating temp binary data file for scenario: citi_bike topology: {self._topology} pid: {pid}. "
"If you want to keep the data, please use MARO CLI command "
f"'maro env data generate -s citi_bike -t {self._topology}' to generate the binary data files first."
)
self._citi_bike_data_pipeline = citi_bike_process.topologies[
self._topology]
self._citi_bike_data_pipeline.download()
self._citi_bike_data_pipeline.clean()
self._citi_bike_data_pipeline.build()
build_folders = self._citi_bike_data_pipeline.get_build_folders()
trip_folder = build_folders["trip"]
weather_folder = build_folders["weather"]
self._conf["weather_data"] = chagne_file_path(
self._conf["weather_data"], weather_folder)
self._conf["trip_data"] = chagne_file_path(self._conf["trip_data"],
trip_folder)
self._conf["stations_init_data"] = chagne_file_path(
self._conf["stations_init_data"], trip_folder)
self._conf["distance_adj_data"] = chagne_file_path(
self._conf["distance_adj_data"], trip_folder)
else:
raise CommandError(
"generate",
f"Can not generate data files for scenario: citi_bike topology: {self._topology}"
)
def download(self, is_force: bool, fall_back: callable = None):
"""Download the original data file.
Args:
is_force(bool): If forced re-download the data file.
fall_back(callable): (optional) Fallback function to execute when download failed.
"""
self._new_folder_list.append(self._download_folder)
os.makedirs(self._download_folder, exist_ok=True)
self._new_file_list.append(self._download_file)
if (not is_force) and os.path.exists(self._download_file):
logger.info_green("File already exists, skipping download.")
else:
logger.info_green(
f"Downloading data from {self._source} to {self._download_file}."
)
try:
download_file(source=self._source,
destination=self._download_file)
except Exception as e:
logger.warning_yellow(
f"Failed to download from {self._source} to {self._download_file}."
)
if fall_back is not None:
logger.warning_yellow(
f"Calling fall_back function: {fall_back}.")
fall_back()
else:
raise CommandError("generate", f"Download error: {e}.")
def download(self, is_force: bool = False):
self._new_folder_list.append(self._download_folder)
os.makedirs(self._download_folder, exist_ok=True)
self._new_file_list.append(self._download_file)
if (not is_force) and os.path.exists(self._download_file):
logger.info_green("File already exists, skipping download.")
else:
logger.info_green(
f"Downloading data from {self._source} to {self._download_file}."
)
try:
download_file(source=self._source,
destination=self._download_file)
except Exception as e:
logger.warning_yellow(
f"Failed to download from {self._source} to {self._download_file}."
)
raise CommandError("generate", f"Download error: {e}.")
# Download text with all urls.
if os.path.exists(self._download_file):
# Download vm_table and cpu_readings
self._aria2p_download(is_force=is_force)
else:
logger.warning(
f"Not found downloaded source file: {self._download_file}.")
def run(command: str) -> str:
logger.debug(command)
completed_process = subprocess.run(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding='utf8')
if completed_process.returncode != 0:
raise CommandError(command, completed_process.stderr)
return completed_process.stdout