def setUpTestData(cls): # Create three GitRepository records repos = ( GitRepository(name="Repo 1", slug="repo-1", remote_url="https://example.com/repo1.git"), GitRepository(name="Repo 2", slug="repo-2", remote_url="https://example.com/repo2.git"), GitRepository(name="Repo 3", slug="repo-3", remote_url="https://example.com/repo3.git"), ) for repo in repos: repo.save(trigger_resync=False) cls.form_data = { "name": "A new Git repository", "slug": "a-new-git-repository", "remote_url": "http://example.com/a_new_git_repository.git", "branch": "develop", "_token": "1234567890abcdef1234567890abcdef", "provided_contents": [ "extras.configcontext", "extras.job", "extras.exporttemplate", ], }
def test_name_contenttype_uniqueness(self): """ The pair of (name, content_type) must be unique for an un-owned ExportTemplate. See GitHub issue #431. """ device_ct = ContentType.objects.get_for_model(Device) ExportTemplate.objects.create(content_type=device_ct, name="Export Template 1", template_code="hello world") with self.assertRaises(ValidationError): duplicate_template = ExportTemplate(content_type=device_ct, name="Export Template 1", template_code="foo") duplicate_template.validated_save() # A differently owned ExportTemplate may have the same name repo = GitRepository( name="Test Git Repository", slug="test-git-repo", remote_url="http://localhost/git.git", username="******", ) repo.save(trigger_resync=False) nonduplicate_template = ExportTemplate(content_type=device_ct, name="Export Template 1", owner=repo, template_code="bar") nonduplicate_template.validated_save()
def test_name_uniqueness(self): """ Verify that two unowned ConfigContexts cannot share the same name (GitHub issue #431). """ ConfigContext.objects.create(name="context 1", weight=100, data={ "a": 123, "b": 456, "c": 777 }) with self.assertRaises(ValidationError): duplicate_context = ConfigContext(name="context 1", weight=200, data={"c": 666}) duplicate_context.validated_save() # If a different context is owned by a GitRepository, that's not considered a duplicate repo = GitRepository( name="Test Git Repository", slug="test-git-repo", remote_url="http://localhost/git.git", username="******", ) repo.save(trigger_resync=False) nonduplicate_context = ConfigContext(name="context 1", weight=300, data={"a": "22"}, owner=repo) nonduplicate_context.validated_save()
class GitRepositoryTest(TransactionTestCase): """ Tests for the GitRepository model class. Note: This is a TransactionTestCase, rather than a TestCase, because the GitRepository save() method uses transaction.on_commit(), which doesn't get triggered in a normal TestCase. """ SAMPLE_TOKEN = "dc6542736e7b02c159d14bc08f972f9ec1e2c45fa" def setUp(self): self.repo = GitRepository( name="Test Git Repository", slug="test-git-repo", remote_url="http://localhost/git.git", username="******", ) self.repo.save(trigger_resync=False) def test_token_rendered(self): self.assertEqual(self.repo.token_rendered, "—") self.repo._token = self.SAMPLE_TOKEN self.assertEqual(self.repo.token_rendered, GitRepository.TOKEN_PLACEHOLDER) self.repo._token = "" self.assertEqual(self.repo.token_rendered, "—") def test_filesystem_path(self): self.assertEqual(self.repo.filesystem_path, os.path.join(settings.GIT_ROOT, self.repo.slug)) def test_save_preserve_token(self): self.repo._token = self.SAMPLE_TOKEN self.repo.save(trigger_resync=False) self.assertEqual(self.repo._token, self.SAMPLE_TOKEN) # As if the user had submitted an "Edit" form, which displays the token placeholder instead of the actual token self.repo._token = GitRepository.TOKEN_PLACEHOLDER self.repo.save(trigger_resync=False) self.assertEqual(self.repo._token, self.SAMPLE_TOKEN) # As if the user had deleted a pre-existing token from the UI self.repo._token = "" self.repo.save(trigger_resync=False) self.assertEqual(self.repo._token, "") def test_verify_user(self): self.assertEqual(self.repo.username, "oauth2") def test_save_relocate_directory(self): with tempfile.TemporaryDirectory() as tmpdirname: with self.settings(GIT_ROOT=tmpdirname): initial_path = self.repo.filesystem_path self.assertIn(self.repo.slug, initial_path) os.makedirs(initial_path) self.repo.slug = "a-new-location" self.repo.save(trigger_resync=False) self.assertFalse(os.path.exists(initial_path)) new_path = self.repo.filesystem_path self.assertIn(self.repo.slug, new_path) self.assertTrue(os.path.isdir(new_path))
def setUp(self): self.repo = GitRepository( name="Test Git Repository", slug="test-git-repo", remote_url="http://localhost/git.git", username="******", ) self.repo.save(trigger_resync=False)
def test_related_object(self): """Test that the `related_object` property is computed properly.""" # Case 1: Job, identified by class_path. with self.settings(JOBS_ROOT=os.path.join(settings.BASE_DIR, "extras/tests/dummy_jobs")): job_class = get_job("local/test_pass/TestPass") job_result = JobResult( name=job_class.class_path, obj_type=ContentType.objects.get(app_label="extras", model="job"), job_id=uuid.uuid4(), ) # Can't just do self.assertEqual(job_result.related_object, job_class) here for some reason self.assertEqual(type(job_result.related_object), type) self.assertTrue(issubclass(job_result.related_object, Job)) self.assertEqual(job_result.related_object.class_path, "local/test_pass/TestPass") job_result.name = "local/no_such_job/NoSuchJob" self.assertIsNone(job_result.related_object) job_result.name = "not-a-class-path" self.assertIsNone(job_result.related_object) # Case 2: GitRepository, identified by name. repo = GitRepository( name="Test Git Repository", slug="test-git-repo", remote_url="http://localhost/git.git", username="******", ) repo.save(trigger_resync=False) job_result = JobResult( name=repo.name, obj_type=ContentType.objects.get_for_model(repo), job_id=uuid.uuid4(), ) self.assertEqual(job_result.related_object, repo) job_result.name = "No such GitRepository" self.assertIsNone(job_result.related_object) # Case 3: Related object with no name, identified by PK/ID ip_address = IPAddress.objects.create(address="1.1.1.1/32") job_result = JobResult( name="irrelevant", obj_type=ContentType.objects.get_for_model(ip_address), job_id=ip_address.pk, ) self.assertEqual(job_result.related_object, ip_address) job_result.job_id = uuid.uuid4() self.assertIsNone(job_result.related_object)
def setUpTestData(cls): cls.repos = ( GitRepository(name="Repo 1", slug="repo-1", remote_url="https://example.com/repo1.git"), GitRepository(name="Repo 2", slug="repo-2", remote_url="https://example.com/repo2.git"), GitRepository(name="Repo 3", slug="repo-3", remote_url="https://example.com/repo3.git"), ) for repo in cls.repos: repo.save(trigger_resync=False)
def create_helper_repo(name="foobaz", provides=None): """ Create a backup and/or intended repo to test helper functions. """ content_provides = f"nautobot_golden_config.{provides}" git_repo = GitRepository( name=name, slug=slugify(name), remote_url=f"http://www.remote-repo.com/{name}.git", branch="main", username="******", provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") if entry.content_identifier == content_provides ], ) git_repo.save(trigger_resync=False)
def setUp(self): self.user = User.objects.create_user(username="******") self.factory = RequestFactory() self.dummy_request = self.factory.get("/no-op/") self.dummy_request.user = self.user # Needed for use with the change_logging decorator self.dummy_request.id = uuid.uuid4() self.site = Site.objects.create(name="Test Site", slug="test-site") self.manufacturer = Manufacturer.objects.create(name="Acme", slug="acme") self.device_type = DeviceType.objects.create( manufacturer=self.manufacturer, model="Frobozz 1000", slug="frobozz1000") self.role = DeviceRole.objects.create(name="router", slug="router") self.device_status = Status.objects.get_for_model(Device).get( slug="active") self.device = Device.objects.create( name="test-device", device_role=self.role, device_type=self.device_type, site=self.site, status=self.device_status, ) self.repo = GitRepository( name="Test Git Repository", slug="test_git_repo", remote_url="http://localhost/git.git", # Provide everything we know we can provide provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") ], ) self.repo.save(trigger_resync=False) self.job_result = JobResult( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), )
def setUpTestData(cls): # Create Three GitRepository records repos = ( GitRepository( name="Repo 1", slug="repo-1", branch="main", provided_contents=[ "extras.configcontext", ], remote_url="https://example.com/repo1.git", ), GitRepository( name="Repo 2", slug="repo-2", branch="develop", provided_contents=[ "extras.configcontext", "extras.job", ], remote_url="https://example.com/repo2.git", ), GitRepository( name="Repo 3", slug="repo-3", branch="next", provided_contents=[ "extras.configcontext", "extras.job", "extras.exporttemplate", ], remote_url="https://example.com/repo3.git", ), ) for repo in repos: repo.save(trigger_resync=False)
def create_git_repos() -> None: """Create five instances of Git Repos. Two GitRepository objects provide Backups. Two GitRepository objects provide Intended. One GitRepository objects provide Jinja Templates. The provided content is matched through a loop, in order to prevent any errors if object ID's change. """ name = "test-backup-repo-1" provides = "nautobot_golden_config.backupconfigs" git_repo_1 = GitRepository( name=name, slug=slugify(name), remote_url=f"http://www.remote-repo.com/{name}.git", branch="main", username="******", provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") if entry.content_identifier == provides ], ) git_repo_1.save(trigger_resync=False) name = "test-backup-repo-2" provides = "nautobot_golden_config.backupconfigs" git_repo_2 = GitRepository( name=name, slug=slugify(name), remote_url=f"http://www.remote-repo.com/{name}.git", branch="main", username="******", provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") if entry.content_identifier == provides ], ) git_repo_2.save(trigger_resync=False) name = "test-intended-repo-1" provides = "nautobot_golden_config.intendedconfigs" git_repo_3 = GitRepository( name=name, slug=slugify(name), remote_url=f"http://www.remote-repo.com/{name}.git", branch="main", username="******", provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") if entry.content_identifier == provides ], ) git_repo_3.save(trigger_resync=False) name = "test-intended-repo-2" provides = "nautobot_golden_config.intendedconfigs" git_repo_4 = GitRepository( name=name, slug=slugify(name), remote_url=f"http://www.remote-repo.com/{name}.git", branch="main", username="******", provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") if entry.content_identifier == provides ], ) git_repo_4.save(trigger_resync=False) name = "test-jinja-repo-1" provides = "nautobot_golden_config.jinjatemplate" git_repo_5 = GitRepository( name=name, slug=slugify(name), remote_url=f"http://www.remote-repo.com/{name}.git", branch="main", username="******", provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") if entry.content_identifier == provides ], ) git_repo_5.save(trigger_resync=False)
class GitTest(TestCase): COMMIT_HEXSHA = "88dd9cd78df89e887ee90a1d209a3e9a04e8c841" def setUp(self): self.user = User.objects.create_user(username="******") self.factory = RequestFactory() self.dummy_request = self.factory.get("/no-op/") self.dummy_request.user = self.user # Needed for use with the change_logging decorator self.dummy_request.id = uuid.uuid4() self.site = Site.objects.create(name="Test Site", slug="test-site") self.manufacturer = Manufacturer.objects.create(name="Acme", slug="acme") self.device_type = DeviceType.objects.create( manufacturer=self.manufacturer, model="Frobozz 1000", slug="frobozz1000") self.role = DeviceRole.objects.create(name="router", slug="router") self.device_status = Status.objects.get_for_model(Device).get( slug="active") self.device = Device.objects.create( name="test-device", device_role=self.role, device_type=self.device_type, site=self.site, status=self.device_status, ) self.repo = GitRepository( name="Test Git Repository", slug="test_git_repo", remote_url="http://localhost/git.git", # Provide everything we know we can provide provided_contents=[ entry.content_identifier for entry in get_datasource_contents("extras.gitrepository") ], ) self.repo.save(trigger_resync=False) self.job_result = JobResult( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) def test_pull_git_repository_and_refresh_data_with_no_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeeed if the given repo is empty. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) self.repo.refresh_from_db() self.assertEqual(self.repo.current_head, self.COMMIT_HEXSHA, self.job_result.data) # TODO: inspect the logs in job_result.data? def test_pull_git_repository_and_refresh_data_with_valid_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeed if valid data is present in the repo. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs( os.path.join(path, "export_templates", "dcim", "device")) with open( os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Region NYC servers", "weight": 1500, "description": "NTP servers for region NYC", "is_active": True, }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file config_context = ConfigContext.objects.get( name="Region NYC servers", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual("NTP servers for region NYC", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) # Make sure Device local config context was successfully populated from file device = Device.objects.get(name=self.device.name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) # Make sure ExportTemplate was successfully loaded from file export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template.j2", ) self.assertIsNotNone(export_template) # Now "resync" the repository, but now those files no longer exist in the repository def empty_repo(path, url): os.remove( os.path.join(path, "config_contexts", "context.yaml")) os.remove( os.path.join(path, "config_contexts", "devices", "test-device.json")) os.remove( os.path.join(path, "export_templates", "dcim", "device", "template.j2")) return mock.DEFAULT MockGitRepo.side_effect = empty_repo # For verisimilitude, don't re-use the old request and job_result self.dummy_request.id = uuid.uuid4() self.job_result = JobResult( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Verify that objects have been removed from the database self.assertEqual( [], list( ConfigContext.objects.filter( owner_content_type=ContentType.objects. get_for_model(GitRepository), owner_object_id=self.repo.pk, )), ) self.assertEqual( [], list( ExportTemplate.objects.filter( owner_content_type=ContentType.objects. get_for_model(GitRepository), owner_object_id=self.repo.pk, )), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner) def test_pull_git_repository_and_refresh_data_with_bad_data( self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should gracefully handle bad data in the Git repository """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs( os.path.join(path, "export_templates", "nosuchapp", "device")) os.makedirs( os.path.join(path, "export_templates", "dcim", "nosuchmodel")) # Malformed JSON with open( os.path.join(path, "config_contexts", "context.json"), "w") as fd: fd.write('{"data": ') # Valid JSON but missing required keys with open( os.path.join(path, "config_contexts", "context2.json"), "w") as fd: fd.write("{}") # No such device with open( os.path.join(path, "config_contexts", "devices", "nosuchdevice.json"), "w", ) as fd: fd.write("{}") # Invalid paths with open( os.path.join( path, "export_templates", "nosuchapp", "device", "template.j2", ), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) with open( os.path.join( path, "export_templates", "dcim", "nosuchmodel", "template.j2", ), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_FAILED, self.job_result.data, ) def test_delete_git_repository_cleanup(self, MockGitRepo): """ When deleting a GitRepository record, the data that it owned should also be deleted. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs( os.path.join(path, "config_contexts", "devices")) os.makedirs( os.path.join(path, "export_templates", "dcim", "device")) with open( os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Region NYC servers", "weight": 1500, "description": "NTP servers for region NYC", "is_active": True, }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write( "{% for device in queryset %}\n{{ device.name }}\n{% endfor %}" ) return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA pull_git_repository_and_refresh_data(self.repo.pk, self.dummy_request, self.job_result) self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file config_context = ConfigContext.objects.get( name="Region NYC servers", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual("NTP servers for region NYC", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) # Make sure Device local config context was successfully populated from file device = Device.objects.get(name=self.device.name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) # Make sure ExportTemplate was successfully loaded from file export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template.j2", ) self.assertIsNotNone(export_template) # Now delete the GitRepository self.repo.delete() with self.assertRaises(ConfigContext.DoesNotExist): config_context = ConfigContext.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) with self.assertRaises(ExportTemplate.DoesNotExist): export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model( GitRepository), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner)
def setUp(self): # Repopulate custom statuses between test cases, as TransactionTestCase deletes them during cleanup create_custom_statuses(None, verbosity=0) self.user = User.objects.create_user(username="******") self.factory = RequestFactory() self.mock_request = self.factory.get("/no-op/") self.mock_request.user = self.user # Needed for use with the change_logging decorator self.mock_request.id = uuid.uuid4() self.site = Site.objects.create(name="Test Site", slug="test-site") self.manufacturer = Manufacturer.objects.create(name="Acme", slug="acme") self.device_type = DeviceType.objects.create( manufacturer=self.manufacturer, model="Frobozz 1000", slug="frobozz1000" ) self.role = DeviceRole.objects.create(name="router", slug="router") self.device_status = Status.objects.get_for_model(Device).get(slug="active") self.device = Device.objects.create( name="test-device", device_role=self.role, device_type=self.device_type, site=self.site, status=self.device_status, ) self.repo = GitRepository( name="Test Git Repository", slug="test_git_repo", remote_url="http://localhost/git.git", # Provide everything we know we can provide provided_contents=[entry.content_identifier for entry in get_datasource_contents("extras.gitrepository")], ) self.repo.save(trigger_resync=False) self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) self.config_context_schema = { "_metadata": { "name": "Config Context Schema 1", "description": "Schema for defining first names, last names and ages.", }, "data_schema": { "title": "Person", "type": "object", "properties": { "firstName": { "type": "string", "description": "The person's first name.", }, "lastName": { "type": "string", "description": "The person's last name.", }, "age": { "description": "Age in years which must be equal to or greater than zero.", "type": "integer", "minimum": 0, }, }, }, }
class GitTest(TransactionTestCase): """ Tests for Git repository handling. This is a TransactionTestCase because it involves JobResult logging. """ databases = ("default", "job_logs") COMMIT_HEXSHA = "88dd9cd78df89e887ee90a1d209a3e9a04e8c841" def setUp(self): # Repopulate custom statuses between test cases, as TransactionTestCase deletes them during cleanup create_custom_statuses(None, verbosity=0) self.user = User.objects.create_user(username="******") self.factory = RequestFactory() self.mock_request = self.factory.get("/no-op/") self.mock_request.user = self.user # Needed for use with the change_logging decorator self.mock_request.id = uuid.uuid4() self.site = Site.objects.create(name="Test Site", slug="test-site") self.manufacturer = Manufacturer.objects.create(name="Acme", slug="acme") self.device_type = DeviceType.objects.create( manufacturer=self.manufacturer, model="Frobozz 1000", slug="frobozz1000" ) self.role = DeviceRole.objects.create(name="router", slug="router") self.device_status = Status.objects.get_for_model(Device).get(slug="active") self.device = Device.objects.create( name="test-device", device_role=self.role, device_type=self.device_type, site=self.site, status=self.device_status, ) self.repo = GitRepository( name="Test Git Repository", slug="test_git_repo", remote_url="http://localhost/git.git", # Provide everything we know we can provide provided_contents=[entry.content_identifier for entry in get_datasource_contents("extras.gitrepository")], ) self.repo.save(trigger_resync=False) self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) self.config_context_schema = { "_metadata": { "name": "Config Context Schema 1", "description": "Schema for defining first names, last names and ages.", }, "data_schema": { "title": "Person", "type": "object", "properties": { "firstName": { "type": "string", "description": "The person's first name.", }, "lastName": { "type": "string", "description": "The person's last name.", }, "age": { "description": "Age in years which must be equal to or greater than zero.", "type": "integer", "minimum": 0, }, }, }, } def populate_repo(self, path, url, *args, **kwargs): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs(os.path.join(path, "config_contexts", "devices")) os.makedirs(os.path.join(path, "config_context_schemas")) os.makedirs(os.path.join(path, "export_templates", "dcim", "device")) os.makedirs(os.path.join(path, "export_templates", "ipam", "vlan")) with open(os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Frobozz 1000 NTP servers", "weight": 1500, "description": "NTP servers for Frobozz 1000 devices **only**", "is_active": True, "schema": "Config Context Schema 1", "device_types": [{"slug": self.device_type.slug}], }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open(os.path.join(path, "config_context_schemas", "schema-1.yaml"), "w") as fd: yaml.dump(self.config_context_schema, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write("{% for device in queryset %}\n{{ device.name }}\n{% endfor %}") with open( os.path.join(path, "export_templates", "dcim", "device", "template2.html"), "w", ) as fd: fd.write("<!DOCTYPE html>/n{% for device in queryset %}\n{{ device.name }}\n{% endfor %}") with open( os.path.join(path, "export_templates", "ipam", "vlan", "template.j2"), "w", ) as fd: fd.write("{% for vlan in queryset %}\n{{ vlan.name }}\n{% endfor %}") return mock.DEFAULT def empty_repo(self, path, url, *args, **kwargs): os.remove(os.path.join(path, "config_contexts", "context.yaml")) os.remove(os.path.join(path, "config_contexts", "devices", "test-device.json")) os.remove(os.path.join(path, "config_context_schemas", "schema-1.yaml")) os.remove(os.path.join(path, "export_templates", "dcim", "device", "template.j2")) os.remove(os.path.join(path, "export_templates", "dcim", "device", "template2.html")) os.remove(os.path.join(path, "export_templates", "ipam", "vlan", "template.j2")) return mock.DEFAULT def assert_config_context_schema_record_exists(self, name): """Helper Func to assert ConfigContextSchema with name=name exists""" config_context_schema_record = ConfigContextSchema.objects.get( name=name, owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) config_context_schema = self.config_context_schema config_context_schema_metadata = config_context_schema["_metadata"] self.assertIsNotNone(config_context_schema_record) self.assertEqual(config_context_schema_metadata["name"], config_context_schema_record.name) self.assertEqual(config_context_schema["data_schema"], config_context_schema_record.data_schema) def assert_device_exists(self, name): """Helper function to assert device exists""" device = Device.objects.get(name=name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) def assert_export_template_device(self, name): export_template_device = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), content_type=ContentType.objects.get_for_model(Device), name=name, ) self.assertIsNotNone(export_template_device) self.assertEqual(export_template_device.mime_type, "text/plain") def assert_config_context_exists(self, name): """Helper function to assert ConfigContext exists""" config_context = ConfigContext.objects.get( name=name, owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual("NTP servers for Frobozz 1000 devices **only**", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual(list(config_context.device_types.all()), [self.device_type]) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) self.assertEqual(self.config_context_schema["_metadata"]["name"], config_context.schema.name) def assert_export_template_html_exist(self, name): """Helper function to assert ExportTemplate exists""" export_template_html = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), content_type=ContentType.objects.get_for_model(Device), name=name, ) self.assertIsNotNone(export_template_html) self.assertEqual(export_template_html.mime_type, "text/html") def assert_export_template_vlan_exists(self, name): """Helper function to assert ExportTemplate exists""" export_template_vlan = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), content_type=ContentType.objects.get_for_model(VLAN), name=name, ) self.assertIsNotNone(export_template_vlan) def test_pull_git_repository_and_refresh_data_with_no_data(self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should succeed if the given repo is empty. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) self.repo.refresh_from_db() self.assertEqual(self.repo.current_head, self.COMMIT_HEXSHA, self.job_result.data) MockGitRepo.assert_called_with(os.path.join(tempdir, self.repo.slug), "http://localhost/git.git") # TODO: inspect the logs in job_result.data? def test_pull_git_repository_and_refresh_data_with_token(self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should correctly make use of a token. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Check that token-based authentication is handled as expected self.repo._token = "1:3@/?=ab@" self.repo.save(trigger_resync=False) # For verisimilitude, don't re-use the old request and job_result self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) MockGitRepo.assert_called_with( os.path.join(tempdir, self.repo.slug), "http://1%3A3%40%2F%3F%3Dab%40@localhost/git.git" ) def test_pull_git_repository_and_refresh_data_with_username_and_token(self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should correctly make use of a username + token. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Check that username/password authentication is handled as expected self.repo.username = "******" self.repo._token = "1:3@/?=ab@" self.repo.save(trigger_resync=False) # For verisimilitude, don't re-use the old request and job_result self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) MockGitRepo.assert_called_with( os.path.join(tempdir, self.repo.slug), "http://n%C3%BA%C3%B1ez:1%3A3%40%2F%3F%3Dab%40@localhost/git.git", ) def test_pull_git_repository_and_refresh_data_with_secrets(self, MockGitRepo): """ The pull_git_repository_and_refresh_data job should correctly make use of secrets. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA with open(os.path.join(tempdir, "username.txt"), "wt") as handle: handle.write("user1234") with open(os.path.join(tempdir, "token.txt"), "wt") as handle: handle.write("1234abcd5678ef90") username_secret = Secret.objects.create( name="Git Username", slug="git-username", provider="text-file", parameters={"path": os.path.join(tempdir, "username.txt")}, ) token_secret = Secret.objects.create( name="Git Token", slug="git-token", provider="text-file", parameters={"path": os.path.join(tempdir, "token.txt")}, ) secrets_group = SecretsGroup.objects.create(name="Git Credentials", slug="git-credentials") SecretsGroupAssociation.objects.create( secret=username_secret, group=secrets_group, access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, secret_type=SecretsGroupSecretTypeChoices.TYPE_USERNAME, ) SecretsGroupAssociation.objects.create( secret=token_secret, group=secrets_group, access_type=SecretsGroupAccessTypeChoices.TYPE_HTTP, secret_type=SecretsGroupSecretTypeChoices.TYPE_TOKEN, ) self.repo.secrets_group = secrets_group self.repo.save(trigger_resync=False) self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) MockGitRepo.assert_called_with( os.path.join(tempdir, self.repo.slug), "http://*****:*****@localhost/git.git", ) def test_pull_git_repository_and_refresh_data_with_valid_data(self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should succeed if valid data is present in the repo. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): MockGitRepo.side_effect = self.populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file self.assert_config_context_exists("Frobozz 1000 NTP servers") # Make sure ConfigContextSchema was successfully loaded from file self.assert_config_context_schema_record_exists("Config Context Schema 1") # Make sure Device local config context was successfully populated from file self.assert_device_exists(self.device.name) # Make sure ExportTemplate was successfully loaded from file self.assert_export_template_device("template.j2") self.assert_export_template_html_exist("template2.html") # Make sure ExportTemplate was successfully loaded from file # Case when ContentType.model != ContentType.name, template was added and deleted during sync (#570) self.assert_export_template_vlan_exists("template.j2") # Now "resync" the repository, but now those files no longer exist in the repository MockGitRepo.side_effect = self.empty_repo # For verisimilitude, don't re-use the old request and job_result self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Verify that objects have been removed from the database self.assertEqual( [], list( ConfigContext.objects.filter( owner_content_type=ContentType.objects.get_for_model(GitRepository), owner_object_id=self.repo.pk, ) ), ) self.assertEqual( [], list( ExportTemplate.objects.filter( owner_content_type=ContentType.objects.get_for_model(GitRepository), owner_object_id=self.repo.pk, ) ), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner) def test_pull_git_repository_and_refresh_data_with_bad_data(self, MockGitRepo): """ The test_pull_git_repository_and_refresh_data job should gracefully handle bad data in the Git repository """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs(os.path.join(path, "config_contexts", "devices")) os.makedirs(os.path.join(path, "config_context_schemas")) os.makedirs(os.path.join(path, "export_templates", "nosuchapp", "device")) os.makedirs(os.path.join(path, "export_templates", "dcim", "nosuchmodel")) # Malformed JSON with open(os.path.join(path, "config_contexts", "context.json"), "w") as fd: fd.write('{"data": ') # Valid JSON but missing required keys with open(os.path.join(path, "config_contexts", "context2.json"), "w") as fd: fd.write("{}") with open(os.path.join(path, "config_context_schemas", "schema-1.yaml"), "w") as fd: fd.write('{"data": ') # Valid JSON but missing required keys with open(os.path.join(path, "config_context_schemas", "schema-2.yaml"), "w") as fd: fd.write("{}") # No such device with open( os.path.join(path, "config_contexts", "devices", "nosuchdevice.json"), "w", ) as fd: fd.write("{}") # Invalid paths with open( os.path.join( path, "export_templates", "nosuchapp", "device", "template.j2", ), "w", ) as fd: fd.write("{% for device in queryset %}\n{{ device.name }}\n{% endfor %}") with open( os.path.join( path, "export_templates", "dcim", "nosuchmodel", "template.j2", ), "w", ) as fd: fd.write("{% for device in queryset %}\n{{ device.name }}\n{% endfor %}") return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_FAILED, self.job_result.data, ) def test_delete_git_repository_cleanup(self, MockGitRepo): """ When deleting a GitRepository record, the data that it owned should also be deleted. """ with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def populate_repo(path, url): os.makedirs(path) # Just make config_contexts and export_templates directories as we don't load jobs os.makedirs(os.path.join(path, "config_contexts")) os.makedirs(os.path.join(path, "config_contexts", "devices")) os.makedirs(os.path.join(path, "config_context_schemas")) os.makedirs(os.path.join(path, "export_templates", "dcim", "device")) with open(os.path.join(path, "config_contexts", "context.yaml"), "w") as fd: yaml.dump( { "_metadata": { "name": "Region NYC servers", "weight": 1500, "description": "NTP servers for region NYC", "is_active": True, "schema": "Config Context Schema 1", }, "ntp-servers": ["172.16.10.22", "172.16.10.33"], }, fd, ) with open( os.path.join(path, "config_contexts", "devices", "test-device.json"), "w", ) as fd: json.dump({"dns-servers": ["8.8.8.8"]}, fd) with open(os.path.join(path, "config_context_schemas", "schema-1.yaml"), "w") as fd: yaml.dump(self.config_context_schema, fd) with open( os.path.join(path, "export_templates", "dcim", "device", "template.j2"), "w", ) as fd: fd.write("{% for device in queryset %}\n{{ device.name }}\n{% endfor %}") return mock.DEFAULT MockGitRepo.side_effect = populate_repo MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA # Run the Git operation and refresh the object from the DB pull_git_repository_and_refresh_data(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual( self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data, ) # Make sure ConfigContext was successfully loaded from file config_context = ConfigContext.objects.get( name="Region NYC servers", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) self.assertIsNotNone(config_context) self.assertEqual(1500, config_context.weight) self.assertEqual("NTP servers for region NYC", config_context.description) self.assertTrue(config_context.is_active) self.assertEqual( {"ntp-servers": ["172.16.10.22", "172.16.10.33"]}, config_context.data, ) # Make sure ConfigContextSchema was successfully loaded from file config_context_schema_record = ConfigContextSchema.objects.get( name="Config Context Schema 1", owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) self.assertEqual(config_context_schema_record, config_context.schema) config_context_schema = self.config_context_schema config_context_schema_metadata = config_context_schema["_metadata"] self.assertIsNotNone(config_context_schema_record) self.assertEqual(config_context_schema_metadata["name"], config_context_schema_record.name) self.assertEqual(config_context_schema["data_schema"], config_context_schema_record.data_schema) # Make sure Device local config context was successfully populated from file device = Device.objects.get(name=self.device.name) self.assertIsNotNone(device.local_context_data) self.assertEqual({"dns-servers": ["8.8.8.8"]}, device.local_context_data) self.assertEqual(device.local_context_data_owner, self.repo) # Make sure ExportTemplate was successfully loaded from file export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), content_type=ContentType.objects.get_for_model(Device), name="template.j2", ) self.assertIsNotNone(export_template) # Now delete the GitRepository self.repo.delete() with self.assertRaises(ConfigContext.DoesNotExist): config_context = ConfigContext.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) with self.assertRaises(ConfigContextSchema.DoesNotExist): config_context_schema = ConfigContextSchema.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) with self.assertRaises(ExportTemplate.DoesNotExist): export_template = ExportTemplate.objects.get( owner_object_id=self.repo.pk, owner_content_type=ContentType.objects.get_for_model(GitRepository), ) device = Device.objects.get(name=self.device.name) self.assertIsNone(device.local_context_data) self.assertIsNone(device.local_context_data_owner) def test_git_dry_run(self, MockGitRepo): with tempfile.TemporaryDirectory() as tempdir: with self.settings(GIT_ROOT=tempdir): def create_empty_repo(path, url, clone_initially=False): os.makedirs(path, exist_ok=True) return mock.DEFAULT MockGitRepo.side_effect = create_empty_repo self.mock_request.id = uuid.uuid4() self.job_result = JobResult.objects.create( name=self.repo.name, obj_type=ContentType.objects.get_for_model(GitRepository), job_id=uuid.uuid4(), ) git_repository_diff_origin_and_local(self.repo.pk, self.mock_request, self.job_result.pk) self.job_result.refresh_from_db() self.assertEqual(self.job_result.status, JobResultStatusChoices.STATUS_COMPLETED, self.job_result.data) MockGitRepo.return_value.checkout.assert_not_called() MockGitRepo.assert_called_with( os.path.join(tempdir, self.repo.slug), self.repo.remote_url, clone_initially=False, ) MockGitRepo.return_value.diff_remote.assert_called()