def test_pull_git_repository_and_refresh_data_with_no_data(self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should succeed if the given repo is empty. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) self.repo.refresh_from_db() self.assertEqual(self.repo.current_head, self.COMMIT_HEXSHA, self.job_result.data) MockGitRepo.assert_called_with(os.path.join(tempdir, self.repo.slug), "http://localhost/git.git")
def test_pull_git_repository_and_refresh_data_with_no_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeeed if the given repo is empty. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) self.repo.refresh_from_db() self.assertEqual(self.repo.current_head, self.COMMIT_HEXSHA, self.job_result.data)
def test_pull_git_repository_and_refresh_data_with_username_and_token( self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should correctly make use of a username + token. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Check that username/password authentication is handled as expected self.repo.username = "******" self.repo._token = "1:3@/?=ab@" self.repo.save() # For verisimilitude, don't re-use the old request and job_result self.dummy_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) MockGitRepo.assert_called_with( os.path.join(tempdir, self.repo.slug), "http://n%C3%BA%C3%B1ez:1%3A3%40%2F%3F%3Dab%40@localhost/git.git", )
def test_delete_git_repository_cleanup(self, MockGitRepo): """ When deleting a GitRepository record, the data that it owned should also be deleted. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs( os.path.join(path, "export_templates", "dcim", "device")) with open( os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Region NYC servers", "weight": 1500, "description": "NTP servers for region NYC", "is_active": True, }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file config_context = ConfigContext.objects.get( name="Region NYC servers", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual("NTP servers for region NYC", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) # Make sure Device local config context was successfully populated from file device = Device.objects.get(name=self.device.name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) # Make sure ExportTemplate was successfully loaded from file export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template.j2", ) self.assertIsNotNone(export_template) # Now delete the GitRepository self.repo.delete() with self.assertRaises(ConfigContext.DoesNotExist): config_context = ConfigContext.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) with self.assertRaises(ExportTemplate.DoesNotExist): export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner)
def test_pull_git_repository_and_refresh_data_with_bad_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should gracefully handle bad data in the Git repository """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs( os.path.join(path, "export_templates", "nosuchapp", "device")) os.makedirs( os.path.join(path, "export_templates", "dcim", "nosuchmodel")) # Malformed JSON with open( os.path.join(path, "config_contexts", "context.json"), "w") as fd: fd.write('{"data": ') # Valid JSON but missing required keys with open( os.path.join(path, "config_contexts", "context2.json"), "w") as fd: fd.write("{}") # No such device with open( os.path.join(path, "config_contexts", "devices", "nosuchdevice.json"), "w", ) as fd: fd.write("{}") # Invalid paths with open( os.path.join( path, "export_templates", "nosuchapp", "device", "template.j2", ), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) with open( os.path.join( path, "export_templates", "dcim", "nosuchmodel", "template.j2", ), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_FAILED, self.job_result.data, )
def test_pull_git_repository_and_refresh_data_with_valid_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeed if valid data is present in the repo. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs( os.path.join(path, "export_templates", "dcim", "device")) with open( os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Region NYC servers", "weight": 1500, "description": "NTP servers for region NYC", "is_active": True, }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file config_context = ConfigContext.objects.get( name="Region NYC servers", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual("NTP servers for region NYC", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) # Make sure Device local config context was successfully populated from file device = Device.objects.get(name=self.device.name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) # Make sure ExportTemplate was successfully loaded from file export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template.j2", ) self.assertIsNotNone(export_template) # Now "resync" the repository, but now those files no longer exist in the repository def empty_repo(path, url): os.remove( os.path.join(path, "config_contexts", "context.yaml")) os.remove( os.path.join(path, "config_contexts", "devices", "test-device.json")) os.remove( os.path.join(path, "export_templates", "dcim", "device", "template.j2")) return mock.DEFAULT MockGitRepo.side_effect = empty_repo # For verisimilitude, don't re-use the old request and job_result self.dummy_request.id = uuid.uuid4() self.job_result = JobResult( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Verify that objects have been removed from the database self.assertEqual( [], list( ConfigContext.objects.filter( owner_content_type=ContentType.objects. get_for_model(GitRepository), owner_object_id=self.repo.pk, )), ) self.assertEqual( [], list( ExportTemplate.objects.filter( owner_content_type=ContentType.objects. get_for_model(GitRepository), owner_object_id=self.repo.pk, )), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner)
def test_pull_git_repository_and_refresh_data_with_valid_data(self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeed if valid data is present in the repo. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): MockGitRepo.side_effect = self.populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file self.assert_config_context_exists("Frobozz 1000 NTP servers") # Make sure ConfigContextSchema was successfully loaded from file self.assert_config_context_schema_record_exists("Config Context Schema 1") # Make sure Device local config context was successfully populated from file self.assert_device_exists(self.device.name) # Make sure ExportTemplate was successfully loaded from file self.assert_export_template_device("template.j2") self.assert_export_template_html_exist("template2.html") # Make sure ExportTemplate was successfully loaded from file # Case when ContentType.model != ContentType.name, template was added and deleted during sync (#570) self.assert_export_template_vlan_exists("template.j2") # Now "resync" the repository, but now those files no longer exist in the repository MockGitRepo.side_effect = self.empty_repo # For verisimilitude, don't re-use the old request and job_result self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Verify that objects have been removed from the database self.assertEqual( [], list( ConfigContext.objects.filter( owner_content_type=ContentType.objects.get_for_model(GitRepository), owner_object_id=self.repo.pk, ) ), ) self.assertEqual( [], list( ExportTemplate.objects.filter( owner_content_type=ContentType.objects.get_for_model(GitRepository), owner_object_id=self.repo.pk, ) ), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner)
def test_pull_git_repository_and_refresh_data_with_secrets(self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should correctly make use of secrets. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA with open(os.path.join(tempdir, "username.txt"), "wt") as handle: handle.write("user1234") with open(os.path.join(tempdir, "token.txt"), "wt") as handle: handle.write("1234abcd5678ef90") username_secret = Secret.objects.create( name="Git Username", slug="git-username", provider="text-file", parameters={"path": os.path.join(tempdir, "username.txt")}, ) token_secret = Secret.objects.create( name="Git Token", slug="git-token", provider="text-file", parameters={"path": os.path.join(tempdir, "token.txt")}, ) secrets_group = SecretsGroup.objects.create(name="Git Credentials", slug="git-credentials") SecretsGroupAssociation.objects.create( secret=username_secret, group=secrets_group, access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, ) SecretsGroupAssociation.objects.create( secret=token_secret, group=secrets_group, access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, ) self.repo.secrets_group = secrets_group self.repo.save(trigger_resync=False) self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) MockGitRepo.assert_called_with( os.path.join(tempdir, self.repo.slug), "http://*****:*****@localhost/git.git", )
def test_pull_git_repository_and_refresh_data_with_valid_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeed if valid data is present in the repo. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs(os.path.join(path, "config_context_schemas")) os.makedirs( os.path.join(path, "export_templates", "dcim", "device")) os.makedirs( os.path.join(path, "export_templates", "ipam", "vlan")) with open( os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Frobozz 1000 NTP servers", "weight": 1500, "description": "NTP servers for Frobozz 1000 devices **only**", "is_active": True, "schema": "Config Context Schema 1", "device_types": [{ "slug": self.device_type.slug }], }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open( os.path.join(path, "config_context_schemas", "schema-1.yaml"), "w") as fd: yaml.dump(self.config_context_schema, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) with open( os.path.join(path, "export_templates", "dcim", "device", "template2.html"), "w", ) as fd: fd.write( "<!DOCTYPE html>/n{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) with open( os.path.join(path, "export_templates", "ipam", "vlan", "template.j2"), "w", ) as fd: fd.write( "{% for vlan in queryset %}\n{{ vlan.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file config_context = ConfigContext.objects.get( name="Frobozz 1000 NTP servers", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual( "NTP servers for Frobozz 1000 devices **only**", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual(list(config_context.device_types.all()), [self.device_type]) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) self.assertEqual( self.config_context_schema["_metadata"]["name"], config_context.schema.name) # Make sure ConfigContextSchema was successfully loaded from file config_context_schema_record = ConfigContextSchema.objects.get( name="Config Context Schema 1", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) config_context_schema = self.config_context_schema config_context_schema_metadata = config_context_schema[ "_metadata"] self.assertIsNotNone(config_context_schema_record) self.assertEqual(config_context_schema_metadata["name"], config_context_schema_record.name) self.assertEqual(config_context_schema["data_schema"], config_context_schema_record.data_schema) # Make sure Device local config context was successfully populated from file device = Device.objects.get(name=self.device.name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) # Make sure ExportTemplate was successfully loaded from file export_template_device = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template.j2", ) self.assertIsNotNone(export_template_device) self.assertEqual(export_template_device.mime_type, "text/plain") export_template_html = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template2.html", ) self.assertIsNotNone(export_template_html) self.assertEqual(export_template_html.mime_type, "text/html") # Make sure ExportTemplate was successfully loaded from file # Case when ContentType.model != ContentType.name, template was added and deleted during sync (#570) export_template_vlan = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(VLAN), name="template.j2", ) self.assertIsNotNone(export_template_vlan) # Now "resync" the repository, but now those files no longer exist in the repository def empty_repo(path, url): os.remove( os.path.join(path, "config_contexts", "context.yaml")) os.remove( os.path.join(path, "config_contexts", "devices", "test-device.json")) os.remove( os.path.join(path, "config_context_schemas", "schema-1.yaml")) os.remove( os.path.join(path, "export_templates", "dcim", "device", "template.j2")) os.remove( os.path.join(path, "export_templates", "dcim", "device", "template2.html")) os.remove( os.path.join(path, "export_templates", "ipam", "vlan", "template.j2")) return mock.DEFAULT MockGitRepo.side_effect = empty_repo # For verisimilitude, don't re-use the old request and job_result self.dummy_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Verify that objects have been removed from the database self.assertEqual( [], list( ConfigContext.objects.filter( owner_content_type=ContentType.objects. get_for_model(GitRepository), owner_object_id=self.repo.pk, )), ) self.assertEqual( [], list( ExportTemplate.objects.filter( owner_content_type=ContentType.objects. get_for_model(GitRepository), owner_object_id=self.repo.pk, )), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner)