def _setup(self):
"""Download data and checkout git repository."""
# Acticate gcloud service
start_time = time.time()
utils.setup_python_path(self.site_packages_dir,
self.config.python_path_str)
utils.active_gcloud_service(self.config.gcloud_key_file_url, self.workspace_dir) # pylint: disable=line-too-long
utils.make_dir_if_not_exist(self.root_output_dir)
self.benchmark_execution_time['activate_gcloud_service'] = time.time() - start_time # pylint: disable=line-too-long
# Download data
start_time = time.time()
utils.download_data(utils.parse_data_downloads_str(self.config.root_data_dir, self.config.gcs_downloads_str)) # pylint: disable=line-too-long
utils.download_data(utils.parse_data_downloads_str(self.config.root_data_dir, self.config.data_downloads_str)) # pylint: disable=line-too-long
self.benchmark_execution_time['download_data'] = time.time(
) - start_time
# Checkout git repositories
start_time = time.time()
site_package_info = utils.checkout_git_repos(
self.config.get_git_repos(self.site_packages_dir),
self.config.force_update)
self.benchmark_execution_time['checkout_repository'] = time.time() - start_time # pylint: disable=line-too-long
self.stream_handler = logging.StreamHandler(sys.stdout)
self.stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logging.getLogger().addHandler(self.stream_handler)
return site_package_info
def _setup(self):
"""Download data and checkout git repository."""
# Acticate gcloud service
start_time = time.time()
utils.setup_python_path(self.site_packages_dir,
self.config.python_path_str)
utils.active_gcloud_service(self.config.gcloud_key_file_url,
self.workspace_dir)
utils.make_dir_if_not_exist(self.root_output_dir)
self.benchmark_execution_time['activate_gcloud_service'] = (
time.time() - start_time)
# Download data
start_time = time.time()
utils.download_data(
utils.parse_data_downloads_str(self.config.root_data_dir,
self.config.gcs_downloads_str))
utils.download_data(
utils.parse_data_downloads_str(self.config.root_data_dir,
self.config.data_downloads_str))
self.benchmark_execution_time['download_data'] = time.time(
) - start_time
# Checkout git repositories
start_time = time.time()
site_package_info = utils.checkout_git_repos(
self.config.get_git_repos(self.site_packages_dir),
self.config.use_cached_site_packages)
self.benchmark_execution_time['checkout_repository'] = (time.time() -
start_time)
# Start cloud TPU.
if self.config.tpu_parameters is not None:
start_time = time.time()
utils.setup_tpu(self.config.tpu_parameters)
tpu_info = tpu_runtime_utils.configure_tpu(
self.config.tpu_parameters)
site_package_info['tpu_version'] = tpu_info
self.benchmark_execution_time['start_tpu'] = time.time(
) - start_time
self.stream_handler = logging.StreamHandler(sys.stdout)
self.stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logging.getLogger().addHandler(self.stream_handler)
return site_package_info
def test_parse_data_downloads_str(self):
data_downloads_str = 'url_1;relative_path_1,url_2;relative_path_2'
download_infos = utils.parse_data_downloads_str(
'/root_data_dir', data_downloads_str)
self.assertEqual(2, len(download_infos))
self.assertEqual(download_infos[0], {
'url': 'url_1',
'local_path': '/root_data_dir/relative_path_1'
})
self.assertEqual(download_infos[1], {
'url': 'url_2',
'local_path': '/root_data_dir/relative_path_2'
})