def test_create_trigger(self): fake_repo_name = utils.get_resource_name(resource_type='repo') branch_regexp = 'fake-branch' env_vars = { 'MY_ENV_VAR1': utils.get_resource_name(resource_type='envvar'), 'MY_ENV_VAR2': utils.get_resource_name(resource_type='envvar') } with self.clean_up_cloudbuild_trigger(fake_repo_name): self._cloudbuild_client.create_trigger(self.project_id, fake_repo_name, branch_regexp, env_vars) service = discovery.build('cloudbuild', 'v1', credentials=self.credentials, cache_discovery=False) request = service.projects().triggers().list( projectId=self.project_id) triggers = [] while request: response = request.execute() triggers += response.get('triggers', []) request = service.projects().triggers().list_next( previous_request=request, previous_response=response) trigger_repo_names = [ trigger.get('triggerTemplate').get('repoName') for trigger in triggers ] self.assertIn(fake_repo_name, trigger_repo_names) for trigger in triggers: repo_name = trigger.get('triggerTemplate').get('repoName') if repo_name == fake_repo_name: self.assertDictEqual(env_vars, trigger.get('substitutions'))
def test_create_keyring_and_key(self): # Keyrings and keys cannot be deleted. And they do not have billable # costs or quota limitations. So no resource cleanup is here. See # https://cloud.google.com/kms/docs/object-hierarchy#lifetime keyring_name = utils.get_resource_name(resource_type='keyring') key_name = utils.get_resource_name(resource_type='key') self._cloudkms_client.create_keyring(self.project_id, keyring_name) keyrings = self._cloudkms_client.list_keyrings(self.project_id) self.assertIn(keyring_name, keyrings) self._cloudkms_client.create_key(self.project_id, keyring_name, key_name) keys = self._cloudkms_client.list_keys(self.project_id, keyring_name) self.assertIn(key_name, keys)
def test_deploy_new_app_sync(self): cluster_name = utils.get_resource_name(resource_type='cluster') with open(self.service_account_key_path) as key_file: key_content = key_file.read() secrets = { 'cloudsql': { 'username': '******', 'password': '******' }, 'cloudsql-oauth-credentials': { 'credentials.json': key_content } } with self.clean_up_cluster(cluster_name): with self.clean_up_docker_image(self.image_tag): url = self.deploygke_workflow.deploy_new_app_sync( project_id=self.project_id, cluster_name=cluster_name, app_directory=self.project_dir, app_name=self.project_name, image_name=self.image_tag, secrets=secrets) admin_url = urllib.parse.urljoin(url, '/admin') response = requests.get(admin_url) self.assertIn('Django administration', response.text)
def test_create_service_account_key(self): service_account_id = utils.get_resource_name(resource_type='sa') service_account_email = '{}@{}.iam.gserviceaccount.com'.format( service_account_id, self.project_id) member = 'serviceAccount:{}'.format(service_account_email) with self.delete_service_account(service_account_email): with self.reset_iam_policy(member, self.ROLES): key_data = (self.service_account_workflow. create_service_account_and_key( self.project_id, service_account_id, 'Test Service Account', self.ROLES)) self.assert_valid_service_account_key(json.loads(key_data)) # Assert the service account is created all_service_accounts = self._list_service_accounts() self.assertIn(service_account_email, all_service_accounts) # Assert the service account has correct roles policy = self._get_iam_policy() for role in self.ROLES: find_role = False for binding in policy['bindings']: if binding['role'] == role: find_role = True self.assertIn(member, binding['members']) self.assertTrue(find_role)
def test_create_duplicate_service_account(self): service_account_id = utils.get_resource_name(resource_type='sa') # Assert no exceptions are raised when creating the same # service account twice for _ in range(2): self._service_account_client.create_service_account( self.project_id, service_account_id, 'Test Service Account', self._ROLES)
def test_create_new_project_no_permission(self): project_name = 'New Project' project_id = utils.get_resource_name(resource_type='project') exception_regex = r'.*HttpError 403.*' # The provided credentials object does not have permission to create # projects with self.assertRaisesRegex(errors.HttpError, exception_regex): self._project_workflow.create_project(project_name, project_id)
def setUp(self): super().setUp() self.project_dir = tempfile.mkdtemp() image_name = utils.get_resource_name(resource_type='image') self.image_tag = '/'.join(['gcr.io', self.project_id, image_name]) self.instance_name = utils.get_resource_name( resource_type='sql-instance') self.database_name = utils.get_resource_name(resource_type='db') app_name = 'fake_app' generator = source_generator.DjangoSourceFileGenerator() generator.generate_new(project_id=self.project_id, project_name=self.project_name, app_name=app_name, project_dir=self.project_dir, database_user=self.database_user, database_password=self.database_password, instance_name=self.instance_name, database_name=self.database_name, image_tag=self.image_tag)
def test_serve_static_content(self): bucket_name = utils.get_resource_name('bucket') static_content_dir = os.path.join(self.project_dir, 'static') with self.clean_up_bucket(bucket_name): self._static_content_serve_workflow.serve_static_content( self.project_id, bucket_name, static_content_dir) object_path = 'static/admin/css/base.css' object_url = 'http://storage.googleapis.com/{}/{}'.format( bucket_name, object_path) response = requests.get(object_url) self.assertIn('DJANGO', response.text)
def test_create_new_project_must_exist(self): project_name = 'New Project' project_id = utils.get_resource_name(resource_type='project') exception_regex = r'.*does not exist.*' with self.assertRaisesRegex(_project.ProjectionCreationError, exception_regex): self._project_workflow.create_project( project_name, project_id, project_creation=_project.CreationMode.MUST_EXIST)
def test_list_repos(self): repo_name = utils.get_resource_name(resource_type='repo') full_repo_name = 'projects/{}/repos/{}'.format(self.project_id, repo_name) prev_repos = self._cloudsource_client.list_repos(self.project_id) prev_repo_names = [repo.get('name') for repo in prev_repos] self.assertNotIn(full_repo_name, prev_repo_names) with self.clean_up_repo(repo_name): self._create_repo(self.project_id, repo_name) cur_repos = self._cloudsource_client.list_repos(self.project_id) repo_names = [repo.get('name') for repo in cur_repos] self.assertIn(full_repo_name, repo_names)
def test_set_cors_policy(self): bucket_name = utils.get_resource_name('bucket') with self.clean_up_bucket(bucket_name): self._storage_client.create_bucket(self.project_id, bucket_name) url = 'http://www.example.com' self._storage_client.set_cors_policy(bucket_name, url) client = discovery.build('storage', 'v1', credentials=self.credentials, cache_discovery=False) request = client.buckets().get(bucket=bucket_name) bucket_body = request.execute(num_retries=5) cors_policy = bucket_body.get('cors') self.assertNotEmpty(cors_policy) self.assertIn(url, cors_policy[0].get('origin'))
class GKEDeployAndUpdateE2ETest(test_base.ResourceCleanUpTest): """End to end test for create and deploy new project.""" _CLOUDSQL_ROLES = ('roles/cloudsql.client', 'roles/cloudsql.editor', 'roles/cloudsql.admin') _FAKE_CLOUDSQL_SERVICE_ACCOUNT = { 'id': utils.get_resource_name('sa'), 'name': 'Fake CloudSQL Credentials', 'file_name': 'credentials.json', 'roles': _CLOUDSQL_ROLES } def setUp(self): super().setUp() self.project_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.project_dir) @staticmethod @backoff.on_exception( backoff.expo, requests.exceptions.ConnectionError, max_tries=3) def _get_with_retry(url: str) -> requests.models.Response: return requests.get(url) @unittest.mock.patch('portpicker.pick_unused_port', return_value=5432) def test_deploy_and_update_new_project(self, unused_mock): # Generate unique resource names fake_superuser_name = 'admin' fake_password = '******' cloud_storage_bucket_name = utils.get_resource_name('bucket') django_project_name = utils.get_resource_name('project', delimiter='') # Generate names we hardcode for users image_tag = '/'.join(['gcr.io', self.project_id, django_project_name]) service_account_email = '{}@{}.iam.gserviceaccount.com'.format( self._FAKE_CLOUDSQL_SERVICE_ACCOUNT['id'], self.project_id) member = 'serviceAccount:{}'.format(service_account_email) with self.clean_up_cluster(django_project_name), \ self.clean_up_bucket(cloud_storage_bucket_name), \ self.clean_up_docker_image(image_tag), \ self.delete_service_account(service_account_email), \ self.reset_iam_policy(member, self._CLOUDSQL_ROLES), \ self.clean_up_sql_instance(django_project_name + '-instance'): test_io = io.TestIO() test_io.answers.append(self.project_id) # project_id test_io.password_answers.append(fake_password) # database password # database password again test_io.password_answers.append(fake_password) test_io.answers.append(self.project_dir) # django_directory_path # The Django local directory is created with tempfile.mkdtemp(). # So when we get this prompt, it exists already. We need to # overwrite it. test_io.answers.append('Y') test_io.answers.append(django_project_name) # django_project_name test_io.answers.append('') # django_app_name # django_superuser_login test_io.answers.append(fake_superuser_name) # django_superuser_password test_io.password_answers.append(fake_password) # django_superuser_password again test_io.password_answers.append(fake_password) test_io.answers.append('') # django_superuser_email fake_service_accounts = { 'cloud_sql': [self._FAKE_CLOUDSQL_SERVICE_ACCOUNT] } arguments = types.SimpleNamespace( credentials=self.credentials, use_existing_project=True, bucket_name=cloud_storage_bucket_name, service_accounts=fake_service_accounts, backend='gke') url = new.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) # Setup Selenium chrome_options = options.Options() chrome_options.add_argument('--headless') # No browser driver = webdriver.Chrome(options=chrome_options) # Assert the web app is available driver.get(url) self.assertIn('Hello from the Cloud!', driver.page_source) # Assert the web app admin page is available admin_url = urllib.parse.urljoin(url, '/admin') driver.get(admin_url) self.assertEqual(driver.title, 'Log in | Django site admin') # Log in with superuser name and password username = driver.find_element_by_id('id_username') password = driver.find_element_by_id('id_password') username.send_keys(fake_superuser_name) password.send_keys(fake_password) driver.find_element_by_css_selector( '.submit-row [value="Log in"]').click() self.assertEqual(driver.title, 'Site administration | Django site admin') object_path = 'static/admin/css/base.css' object_url = 'http://storage.googleapis.com/{}/{}'.format( cloud_storage_bucket_name, object_path) response = requests.get(object_url) # Assert the static content is successfully uploaded self.assertIn('DJANGO', response.text) # Assert the deployed app is using static content from the GCS # bucket self.assertIn(cloud_storage_bucket_name, driver.page_source) # Test update command test_io = io.TestIO() test_io.password_answers.append(fake_password) # database password test_io.answers.append(self.project_dir) # django_directory_path view_file_path = os.path.join(self.project_dir, 'home', 'views.py') with open(view_file_path) as view_file: file_content = view_file.read() file_content = file_content.replace('Hello', 'Hello1') with open(view_file_path, 'w') as view_file: view_file.write(file_content) arguments = types.SimpleNamespace( credentials=self.credentials, backend='gke') update.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) # This call is flaky without retry. Sometimes this call is made # after the pod is ready but before the http server is ready. response = self._get_with_retry(url) self.assertIn('Hello1 from the Cloud!', response.text)
def test_cloudify_and_update_new_project(self, unused_mock): # Generate unique resource names fake_superuser_name = 'admin' fake_password = '******' cloud_storage_bucket_name = utils.get_resource_name('bucket') database_instance_name = utils.get_resource_name('sql-instance') service_name = utils.get_resource_name('svc', delimiter='') cluster_name = utils.get_resource_name('cluster') # Generate names we hardcode for users service_account_email = '{}@{}.iam.gserviceaccount.com'.format( self._FAKE_CLOUDSQL_SERVICE_ACCOUNT['id'], self.project_id) member = 'serviceAccount:{}'.format(service_account_email) settings_path = os.path.join(self.project_dir, 'mysite', 'settings.py') requirements_path = os.path.join(self.project_dir, 'requirements.txt') with self.clean_up_cluster(cluster_name), \ self.clean_up_bucket(cloud_storage_bucket_name), \ self.delete_service_account(service_account_email), \ self.reset_iam_policy(member, self._CLOUDSQL_ROLES), \ self.clean_up_sql_instance(database_instance_name), \ self.clean_up_appengine_service(service_name): test_io = e2e_utils.create_cloudify_command_io( self.project_id, self.project_dir, requirements_path, settings_path) fake_service_accounts = { 'cloud_sql': [self._FAKE_CLOUDSQL_SERVICE_ACCOUNT] } arguments = types.SimpleNamespace( credentials=self.credentials, use_existing_project=True, bucket_name=cloud_storage_bucket_name, service_accounts=fake_service_accounts, appengine_service_name=service_name, cluster_name=cluster_name, database_instance_name=database_instance_name, backend='gke') url = cloudify.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) # Setup Selenium chrome_options = options.Options() chrome_options.add_argument('--headless') # No browser driver = webdriver.Chrome(options=chrome_options) # Assert the web app is available driver.get(url) self.assertIn('Hello from the Cloud!', driver.page_source) # Assert the web app admin page is available admin_url = urllib.parse.urljoin(url, '/admin') driver.get(admin_url) self.assertEqual(driver.title, 'Log in | Django site admin') # Log in with superuser name and password username = driver.find_element_by_id('id_username') password = driver.find_element_by_id('id_password') username.send_keys(fake_superuser_name) password.send_keys(fake_password) driver.find_element_by_css_selector( '.submit-row [value="Log in"]').click() self.assertEqual(driver.title, 'Site administration | Django site admin') # Assert the static content is successfully uploaded object_path = 'static/admin/css/base.css' object_url = 'http://storage.googleapis.com/{}/{}'.format( cloud_storage_bucket_name, object_path) response = requests.get(object_url) self.assertIn('DJANGO', response.text) # Assert the deployed app is using static content from the GCS # bucket self.assertIn(cloud_storage_bucket_name, driver.page_source) # Test update command test_io = e2e_utils.create_update_command_io(self.project_dir) view_file_path = os.path.join(self.project_dir, 'polls', 'views.py') with open(view_file_path) as view_file: file_content = view_file.read() file_content = file_content.replace('Hello', 'Hello1') with open(view_file_path, 'w') as view_file: view_file.write(file_content) arguments = types.SimpleNamespace( credentials=self.credentials, cluster_name=cluster_name, database_instance_name=database_instance_name) update.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) # This call is flaky without retry. Sometimes this call is made # after the pod is ready but before the http server is ready. response = e2e_utils.get_with_retry(url) self.assertIn('Hello1 from the Cloud!', response.text)
def test_reuse_cluster(self): cluster_name = utils.get_resource_name(resource_type='cluster') with self.clean_up_cluster(cluster_name): for _ in range(2): self._container_client.create_cluster_sync( self.project_id, cluster_name)
def test_reuse_bucket(self): bucket_name = utils.get_resource_name('bucket') with self.clean_up_bucket(bucket_name): for _ in range(3): self._static_content_serve_client.create_bucket( self.project_id, bucket_name)
def test_deploy_and_update_new_project(self): # Generate unique resource names fake_superuser_name = 'admin' fake_password = '******' cloud_storage_bucket_name = utils.get_resource_name('bucket') django_project_name = utils.get_resource_name('project', delimiter='') # Generate names we hardcode for users image_tag = '/'.join(['gcr.io', self.project_id, django_project_name]) service_account_email = '{}@{}.iam.gserviceaccount.com'.format( self._FAKE_CLOUDSQL_SERVICE_ACCOUNT['id'], self.project_id) member = 'serviceAccount:{}'.format(service_account_email) with self.clean_up_cluster(django_project_name), \ self.clean_up_bucket(cloud_storage_bucket_name), \ self.clean_up_docker_image(image_tag), \ self.delete_service_account(service_account_email), \ self.reset_iam_policy(member, self._CLOUDSQL_ROLES), \ self.clean_up_sql_instance(django_project_name + '-instance'): test_io = io.TestIO() test_io.answers.append(self.project_id) # project_id test_io.password_answers.append(fake_password) # database password # database password again test_io.password_answers.append(fake_password) test_io.answers.append(self.project_dir) # django_directory_path # The Django local directory is created with tempfile.mkdtemp(). # So when we get this prompt, it exists already. We need to # overwrite it. test_io.answers.append('Y') test_io.answers.append(django_project_name) # django_project_name test_io.answers.append('') # django_app_name # django_superuser_login test_io.answers.append(fake_superuser_name) # django_superuser_password test_io.password_answers.append(fake_password) # django_superuser_password again test_io.password_answers.append(fake_password) test_io.answers.append('') # django_superuser_email fake_service_accounts = { 'cloud_sql': [self._FAKE_CLOUDSQL_SERVICE_ACCOUNT] } arguments = types.SimpleNamespace( credentials=self.credentials, use_existing_project=True, bucket_name=cloud_storage_bucket_name, service_accounts=fake_service_accounts, backend='gke') url = new.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) # Setup Selenium chrome_options = options.Options() chrome_options.add_argument('--headless') # No browser driver = webdriver.Chrome(options=chrome_options) # Assert the web app is available driver.get(url) self.assertIn('Hello from the Cloud!', driver.page_source) # Assert the web app admin page is available admin_url = urllib.parse.urljoin(url, '/admin') driver.get(admin_url) self.assertEqual(driver.title, 'Log in | Django site admin') # Log in with superuser name and password username = driver.find_element_by_id('id_username') password = driver.find_element_by_id('id_password') username.send_keys(fake_superuser_name) password.send_keys(fake_password) driver.find_element_by_css_selector( '.submit-row [value="Log in"]').click() self.assertEqual(driver.title, 'Site administration | Django site admin') object_path = 'static/admin/css/base.css' object_url = 'http://storage.googleapis.com/{}/{}'.format( cloud_storage_bucket_name, object_path) response = requests.get(object_url) # Assert the static content is successfully uploaded self.assertIn('DJANGO', response.text) # Assert the deployed app is using static content from the GCS # bucket self.assertIn(cloud_storage_bucket_name, driver.page_source) # Test update command test_io = io.TestIO() test_io.password_answers.append(fake_password) # database password test_io.answers.append(self.project_dir) # django_directory_path view_file_path = os.path.join(self.project_dir, 'home', 'views.py') with open(view_file_path) as view_file: file_content = view_file.read() file_content = file_content.replace('Hello', 'Hello1') with open(view_file_path, 'w') as view_file: view_file.write(file_content) arguments = types.SimpleNamespace(credentials=self.credentials) update.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) response = requests.get(url) self.assertIn('Hello1 from the Cloud!', response.text)
class GAECloudifyAndUpdateE2ETest(test_base.ResourceCleanUp): """End to end test for deploying an existing Django project on GAE.""" _CLOUDSQL_ROLES = ('roles/cloudsql.client', 'roles/cloudsql.editor', 'roles/cloudsql.admin') _FAKE_CLOUDSQL_SERVICE_ACCOUNT = { 'id': utils.get_resource_name('sa'), 'name': 'Fake CloudSQL Credentials', 'file_name': 'credentials.json', 'roles': _CLOUDSQL_ROLES } def setUp(self): super().setUp() self.project_dir = self._get_django_project_path() def _get_django_project_path(self): """Returns the absolute path of an existing Django project. This project exists under django_cloud_deply/tests/e2e/data/basic_django_project """ dirname = os.path.dirname(os.path.abspath(__file__)) return os.path.join(dirname, 'data', 'basic_django_project') @unittest.mock.patch('portpicker.pick_unused_port', return_value=5432) def test_cloudify_and_update_new_project(self, unused_mock): # Generate unique resource names fake_superuser_name = 'admin' fake_password = '******' cloud_storage_bucket_name = utils.get_resource_name('bucket') database_instance_name = utils.get_resource_name('sql-instance') service_name = utils.get_resource_name('svc', delimiter='') # Generate names we hardcode for users service_account_email = '{}@{}.iam.gserviceaccount.com'.format( self._FAKE_CLOUDSQL_SERVICE_ACCOUNT['id'], self.project_id) member = 'serviceAccount:{}'.format(service_account_email) settings_path = os.path.join(self.project_dir, 'mysite', 'settings.py') requirements_path = os.path.join(self.project_dir, 'requirements.txt') with self.clean_up_bucket(cloud_storage_bucket_name), \ self.delete_service_account(service_account_email), \ self.reset_iam_policy(member, self._CLOUDSQL_ROLES), \ self.clean_up_sql_instance(database_instance_name), \ self.clean_up_appengine_service(service_name): test_io = e2e_utils.create_cloudify_command_io( self.project_id, self.project_dir, requirements_path, settings_path) fake_service_accounts = { 'cloud_sql': [self._FAKE_CLOUDSQL_SERVICE_ACCOUNT] } arguments = types.SimpleNamespace( credentials=self.credentials, use_existing_project=True, bucket_name=cloud_storage_bucket_name, service_accounts=fake_service_accounts, appengine_service_name=service_name, database_instance_name=database_instance_name, backend='gae') url = cloudify.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) # Setup Selenium chrome_options = options.Options() chrome_options.add_argument('--headless') # No browser driver = webdriver.Chrome(options=chrome_options) # Assert the web app is available driver.get(url) self.assertIn('Hello from the Cloud!', driver.page_source) # Assert the web app admin page is available admin_url = urllib.parse.urljoin(url, '/admin') driver.get(admin_url) self.assertEqual(driver.title, 'Log in | Django site admin') # Log in with superuser name and password username = driver.find_element_by_id('id_username') password = driver.find_element_by_id('id_password') username.send_keys(fake_superuser_name) password.send_keys(fake_password) driver.find_element_by_css_selector( '.submit-row [value="Log in"]').click() self.assertEqual(driver.title, 'Site administration | Django site admin') # Assert the static content is successfully uploaded object_path = 'static/admin/css/base.css' object_url = 'http://storage.googleapis.com/{}/{}'.format( cloud_storage_bucket_name, object_path) response = requests.get(object_url) self.assertIn('DJANGO', response.text) # Assert the deployed app is using static content from the GCS # bucket self.assertIn(cloud_storage_bucket_name, driver.page_source) # Test update command test_io = e2e_utils.create_update_command_io(self.project_dir) view_file_path = os.path.join(self.project_dir, 'polls', 'views.py') with open(view_file_path) as view_file: file_content = view_file.read() file_content = file_content.replace('Hello', 'Hello1') with open(view_file_path, 'w') as view_file: view_file.write(file_content) arguments = types.SimpleNamespace( credentials=self.credentials, database_instance_name=database_instance_name) update.main(arguments, test_io) # Assert answers are all used. self.assertEqual(len(test_io.answers), 0) self.assertEqual(len(test_io.password_answers), 0) e2e_utils.wait_for_appengine_update_ready(self.project_id, service_name) response = e2e_utils.get_with_retry(url) self.assertIn('Hello1 from the Cloud!', response.text)