def _setup(self): """Download data and checkout git repository.""" # Acticate gcloud service start_time = time.time() utils.setup_python_path(self.site_packages_dir, self.config.python_path_str) utils.active_gcloud_service(self.config.gcloud_key_file_url, self.workspace_dir) # pylint: disable=line-too-long utils.make_dir_if_not_exist(self.root_output_dir) self.benchmark_execution_time['activate_gcloud_service'] = time.time() - start_time # pylint: disable=line-too-long # Download data start_time = time.time() utils.download_data(utils.parse_data_downloads_str(self.config.root_data_dir, self.config.gcs_downloads_str)) # pylint: disable=line-too-long utils.download_data(utils.parse_data_downloads_str(self.config.root_data_dir, self.config.data_downloads_str)) # pylint: disable=line-too-long self.benchmark_execution_time['download_data'] = time.time( ) - start_time # Checkout git repositories start_time = time.time() site_package_info = utils.checkout_git_repos( self.config.get_git_repos(self.site_packages_dir), self.config.force_update) self.benchmark_execution_time['checkout_repository'] = time.time() - start_time # pylint: disable=line-too-long self.stream_handler = logging.StreamHandler(sys.stdout) self.stream_handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logging.getLogger().addHandler(self.stream_handler) return site_package_info
def _setup(self): """Download data and checkout git repository.""" # Acticate gcloud service start_time = time.time() utils.setup_python_path(self.site_packages_dir, self.config.python_path_str) utils.active_gcloud_service(self.config.gcloud_key_file_url, self.workspace_dir) utils.make_dir_if_not_exist(self.root_output_dir) self.benchmark_execution_time['activate_gcloud_service'] = ( time.time() - start_time) # Download data start_time = time.time() utils.download_data( utils.parse_data_downloads_str(self.config.root_data_dir, self.config.gcs_downloads_str)) utils.download_data( utils.parse_data_downloads_str(self.config.root_data_dir, self.config.data_downloads_str)) self.benchmark_execution_time['download_data'] = time.time( ) - start_time # Checkout git repositories start_time = time.time() site_package_info = utils.checkout_git_repos( self.config.get_git_repos(self.site_packages_dir), self.config.use_cached_site_packages) self.benchmark_execution_time['checkout_repository'] = (time.time() - start_time) # Start cloud TPU. if self.config.tpu_parameters is not None: start_time = time.time() utils.setup_tpu(self.config.tpu_parameters) tpu_info = tpu_runtime_utils.configure_tpu( self.config.tpu_parameters) site_package_info['tpu_version'] = tpu_info self.benchmark_execution_time['start_tpu'] = time.time( ) - start_time self.stream_handler = logging.StreamHandler(sys.stdout) self.stream_handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logging.getLogger().addHandler(self.stream_handler) return site_package_info
def _setup(self): """Download data and checkout git repository.""" # Set up the raid array. start_time = time.time() device_utils.create_drive_from_devices('/data', self.config.gce_nvme_raid_str) self.benchmark_execution_time['create_drive'] = time.time( ) - start_time start_time = time.time() utils.download_from_gcs([{ 'gcs_url': 'gs://tf-performance/auth_tokens', 'local_path': os.path.join(self.workspace_dir, 'auth_tokens') }]) self.benchmark_execution_time['download_token'] = time.time( ) - start_time # Acticate gcloud service start_time = time.time() utils.setup_python_path(self.site_packages_dir, self.config.python_path_str) utils.active_gcloud_service(self.auth_token_path) utils.make_dir_if_not_exist(self.output_root_dir) self.benchmark_execution_time['activate_gcloud_service'] = time.time( ) - start_time # Download data start_time = time.time() utils.download_from_gcs(self.config.get_gcs_downloads('/data')) self.benchmark_execution_time['download_data'] = time.time( ) - start_time # Checkout git repositories start_time = time.time() site_package_info = utils.checkout_git_repos( self.config.get_git_repos(self.site_packages_dir), self.config.force_update) self.benchmark_execution_time['checkout_repository'] = time.time( ) - start_time self.stream_handler = logging.StreamHandler(sys.stdout) self.stream_handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s')) logging.getLogger().addHandler(self.stream_handler) return site_package_info
def test_checkout_git_repos(self, run_commands_mock, get_git_repo_info_mock): git_repo_1 = {} git_repo_1['url'] = 'url_1' git_repo_1['local_path'] = 'local_path_1' git_repo_1['dir_name'] = 'dir_name_1' git_repo_1['branch'] = 'branch_1' git_repo_1['git_hash'] = 'git_hash_1' git_repo_2 = {} git_repo_2['url'] = 'url_2' git_repo_2['local_path'] = 'local_path_2' git_repo_2['dir_name'] = 'dir_name_2' git_repo_2['branch'] = 'branch_2' git_repo_info_1 = {'url': 'url_1'} git_repo_info_2 = {'url': 'url_2'} get_git_repo_info_mock.side_effect = \ lambda local_path: git_repo_info_1 if local_path == 'local_path_1' else git_repo_info_2 # pylint: disable=line-too-long site_package_info = utils.checkout_git_repos([git_repo_1, git_repo_2], False) self.assertEqual(2, len(site_package_info)) self.assertEqual(git_repo_info_1, site_package_info['dir_name_1']) self.assertEqual(git_repo_info_2, site_package_info['dir_name_2']) run_commands_mock.assert_has_calls( any_order=False, calls=[ call(['git clone url_1 local_path_1']), call(['git -C local_path_1 checkout branch_1']), call(['git -C local_path_1 pull']), call(['git -C local_path_1 reset --hard git_hash_1']), call(['git clone url_2 local_path_2']), call(['git -C local_path_2 checkout branch_2']) ])