def _setup(self):
"""Download data and checkout git repository."""
# Acticate gcloud service
start_time = time.time()
utils.setup_python_path(self.site_packages_dir,
self.config.python_path_str)
utils.active_gcloud_service(self.config.gcloud_key_file_url, self.workspace_dir) # pylint: disable=line-too-long
utils.make_dir_if_not_exist(self.root_output_dir)
self.benchmark_execution_time['activate_gcloud_service'] = time.time() - start_time # pylint: disable=line-too-long
# Download data
start_time = time.time()
utils.download_data(utils.parse_data_downloads_str(self.config.root_data_dir, self.config.gcs_downloads_str)) # pylint: disable=line-too-long
utils.download_data(utils.parse_data_downloads_str(self.config.root_data_dir, self.config.data_downloads_str)) # pylint: disable=line-too-long
self.benchmark_execution_time['download_data'] = time.time(
) - start_time
# Checkout git repositories
start_time = time.time()
site_package_info = utils.checkout_git_repos(
self.config.get_git_repos(self.site_packages_dir),
self.config.force_update)
self.benchmark_execution_time['checkout_repository'] = time.time() - start_time # pylint: disable=line-too-long
self.stream_handler = logging.StreamHandler(sys.stdout)
self.stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logging.getLogger().addHandler(self.stream_handler)
return site_package_info
def _setup(self):
utils.setup_python_path(self.site_packages_dir, config.python_path_str)
utils.active_gcloud_service(self.auth_token_path)
utils.make_dir_if_not_exist(self.output_root_dir)
self.streamHandler = logging.StreamHandler(sys.stdout)
self.streamHandler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logging.getLogger().addHandler(self.streamHandler)
def _setup(self):
"""Download data and checkout git repository."""
# Acticate gcloud service
start_time = time.time()
utils.setup_python_path(self.site_packages_dir,
self.config.python_path_str)
utils.active_gcloud_service(self.config.gcloud_key_file_url,
self.workspace_dir)
utils.make_dir_if_not_exist(self.root_output_dir)
self.benchmark_execution_time['activate_gcloud_service'] = (
time.time() - start_time)
# Download data
start_time = time.time()
utils.download_data(
utils.parse_data_downloads_str(self.config.root_data_dir,
self.config.gcs_downloads_str))
utils.download_data(
utils.parse_data_downloads_str(self.config.root_data_dir,
self.config.data_downloads_str))
self.benchmark_execution_time['download_data'] = time.time(
) - start_time
# Checkout git repositories
start_time = time.time()
site_package_info = utils.checkout_git_repos(
self.config.get_git_repos(self.site_packages_dir),
self.config.use_cached_site_packages)
self.benchmark_execution_time['checkout_repository'] = (time.time() -
start_time)
# Start cloud TPU.
if self.config.tpu_parameters is not None:
start_time = time.time()
utils.setup_tpu(self.config.tpu_parameters)
tpu_info = tpu_runtime_utils.configure_tpu(
self.config.tpu_parameters)
site_package_info['tpu_version'] = tpu_info
self.benchmark_execution_time['start_tpu'] = time.time(
) - start_time
self.stream_handler = logging.StreamHandler(sys.stdout)
self.stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logging.getLogger().addHandler(self.stream_handler)
return site_package_info
def _setup(self):
"""Download data and checkout git repository."""
# Set up the raid array.
start_time = time.time()
device_utils.create_drive_from_devices('/data',
self.config.gce_nvme_raid_str)
self.benchmark_execution_time['create_drive'] = time.time(
) - start_time
start_time = time.time()
utils.download_from_gcs([{
'gcs_url':
'gs://tf-performance/auth_tokens',
'local_path':
os.path.join(self.workspace_dir, 'auth_tokens')
}])
self.benchmark_execution_time['download_token'] = time.time(
) - start_time
# Acticate gcloud service
start_time = time.time()
utils.setup_python_path(self.site_packages_dir,
self.config.python_path_str)
utils.active_gcloud_service(self.auth_token_path)
utils.make_dir_if_not_exist(self.output_root_dir)
self.benchmark_execution_time['activate_gcloud_service'] = time.time(
) - start_time
# Download data
start_time = time.time()
utils.download_from_gcs(self.config.get_gcs_downloads('/data'))
self.benchmark_execution_time['download_data'] = time.time(
) - start_time
# Checkout git repositories
start_time = time.time()
site_package_info = utils.checkout_git_repos(
self.config.get_git_repos(self.site_packages_dir),
self.config.force_update)
self.benchmark_execution_time['checkout_repository'] = time.time(
) - start_time
self.stream_handler = logging.StreamHandler(sys.stdout)
self.stream_handler.setFormatter(
logging.Formatter('%(asctime)s %(levelname)s: %(message)s'))
logging.getLogger().addHandler(self.stream_handler)
return site_package_info