def test_bad_repository_url(self, mock_requests, mock_clone_repository):
'''
Tests where a bad url is provided for the github repo.
This can either be a url that does not exist OR one that
does not show up since it's private (we don't allow cloning
of private repos)
'''
mock_response = mock.MagicMock()
mock_response.status_code == 404
mock_requests.get.return_value = mock_response
repo_url = 'http://github.com/some-repo/'
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
o = OperationDbModel.objects.create(id=str(op_uuid))
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
with self.assertRaisesRegex(Exception, 'not find'):
perform_operation_ingestion(
repo_url,
str(op_uuid),
None # this means no specific commit was requested
)
mock_clone_repository.assert_not_called()
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 0)
op = OperationDbModel.objects.get(id=str(op_uuid))
self.assertFalse(op.successful_ingestion)
def test_workspace_operation_validates(
self, mock_shutil, mock_read_operation_json, mock_check_for_repo,
mock_checkout_branch, mock_clone_repository,
mock_retrieve_commit_hash, mock_save_operation,
mock_check_required_files, mock_retrieve_repo_name,
mock_prepare_operation):
'''
Here, we do not request a specific commit ID, so different
functions are called
'''
filepath = os.path.join(TESTDIR, 'valid_workspace_operation.json')
fp = open(filepath)
op_dict = json.load(fp)
fp.close()
mock_read_operation_json.return_value = op_dict
mock_hash = 'abcd'
mock_dir = '/some/mock/staging/dir'
mock_clone_repository.return_value = mock_dir
mock_retrieve_commit_hash.return_value = 'abcd'
repo_url = 'http://github.com/some-repo/'
repo_name = 'some-repo'
mock_retrieve_repo_name.return_value = repo_name
mock_commit_id = 'abcdefg1234'
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
o = OperationDbModel.objects.create(id=str(op_uuid))
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
perform_operation_ingestion(
repo_url,
str(op_uuid),
None # this means a specific commit was NOT requested
)
mock_clone_repository.assert_called_with(repo_url)
mock_retrieve_commit_hash.assert_called_with(mock_dir)
mock_save_operation.assert_called()
mock_shutil.rmtree.assert_called_with(mock_dir)
mock_checkout_branch.assert_not_called()
mock_check_for_repo.assert_called_with(repo_url)
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 1)
op = OperationDbModel.objects.get(id=op_uuid)
self.assertTrue(op.workspace_operation)
def test_operation_rejected_for_bad_formatting(
self, mock_shutil, mock_read_operation_json, mock_check_for_repo,
mock_checkout_branch, mock_clone_repository,
mock_retrieve_commit_hash, mock_save_operation,
mock_check_required_files, mock_retrieve_repo_name,
mock_prepare_operation):
'''
Here we check that bad formatting in the case of nested objects
will raise an exception. This stems from a problem where the 'inputs'
dict was missing a brace. This caused the remaining inputs to appear
as extra keys of a single input. However, it was valid json so it all
passed validation. The result was that some of the inputs were missing
in the final ingested operation spec. The extra keys were silently
ignored.
This tests that we raise an exception under these circumstances.
'''
j = json.load(
open(os.path.join(TESTDIR, 'op_with_bad_formatting.json')))
mock_read_operation_json.return_value = j
mock_hash = 'abcd'
mock_dir = '/some/mock/staging/dir'
mock_clone_repository.return_value = mock_dir
mock_retrieve_commit_hash.return_value = 'abcd'
repo_url = 'http://github.com/some-repo/'
repo_name = 'some-repo'
mock_retrieve_repo_name.return_value = repo_name
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
o = OperationDbModel.objects.create(id=str(op_uuid))
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
with self.assertRaises(ValidationError):
perform_operation_ingestion(repo_url, str(op_uuid), None)
mock_clone_repository.assert_called_with(repo_url)
mock_retrieve_commit_hash.assert_called_with(mock_dir)
mock_save_operation.assert_not_called()
mock_shutil.rmtree.assert_called_with(mock_dir)
# no specific commit was passed (None), so this function is not called
mock_checkout_branch.assert_not_called()
mock_check_for_repo.assert_called_with(repo_url)
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 0)
def test_operation_validates(
self, mock_shutil, mock_read_operation_json, mock_check_for_repo,
mock_checkout_branch, mock_clone_repository,
mock_retrieve_commit_hash, mock_save_operation,
mock_check_required_files, mock_retrieve_repo_name,
mock_prepare_operation):
'''
Here, the ingestion request does not include a specific commit, so
we ensure that the specific git commit functions are called or not called
'''
mock_read_operation_json.return_value = self.valid_dict
mock_hash = 'abcd'
mock_dir = '/some/mock/staging/dir'
mock_clone_repository.return_value = mock_dir
mock_retrieve_commit_hash.return_value = 'abcd'
repo_url = 'http://github.com/some-repo/'
repo_name = 'some-repo'
mock_retrieve_repo_name.return_value = repo_name
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
o = OperationDbModel.objects.create(id=str(op_uuid))
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
perform_operation_ingestion(
repo_url,
str(op_uuid),
None # this means no specific commit was requested
)
mock_clone_repository.assert_called_with(repo_url)
mock_retrieve_commit_hash.assert_called_with(mock_dir)
mock_save_operation.assert_called()
mock_shutil.rmtree.assert_called_with(mock_dir)
mock_checkout_branch.assert_not_called()
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 1)
op = OperationDbModel.objects.get(id=op_uuid)
self.assertFalse(op.workspace_operation)
def test_operation_validates_case2(
self, mock_shutil, mock_read_operation_json, mock_check_for_repo,
mock_checkout_branch, mock_clone_repository,
mock_retrieve_commit_hash, mock_save_operation,
mock_check_required_files, mock_retrieve_repo_name,
mock_prepare_operation):
'''
Here, we explicitly request a specific commit ID, so different
functions are called
'''
mock_read_operation_json.return_value = self.valid_dict
mock_hash = 'abcd'
mock_dir = '/some/mock/staging/dir'
mock_clone_repository.return_value = mock_dir
mock_retrieve_commit_hash.return_value = 'abcd'
repo_url = 'http://github.com/some-repo/'
repo_name = 'some-repo'
mock_retrieve_repo_name.return_value = repo_name
mock_commit_id = 'abcd1234'
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
o = OperationDbModel.objects.create(id=str(op_uuid))
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
perform_operation_ingestion(repo_url, str(op_uuid), mock_commit_id)
mock_clone_repository.assert_called_with(repo_url)
mock_retrieve_commit_hash.assert_not_called()
mock_save_operation.assert_called()
mock_shutil.rmtree.assert_called_with(mock_dir)
mock_checkout_branch.assert_called_with(mock_dir, mock_commit_id)
mock_check_for_repo.assert_called_with(repo_url)
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 1)
op = OperationDbModel.objects.get(id=op_uuid)
self.assertFalse(op.workspace_operation)
def ingest_new_operation(operation_uuid_str, repository_url, commit_id):
'''
This function kicks off the ingestion process for a new Operation
`repository_url` is the url to the git repo
`commit_id` is the commit we want to ingest. Can be None, in which case we
will default to the main branch
'''
try:
operation = Operation.objects.get(id=operation_uuid_str)
except Operation.DoesNotExist:
logger.error('Could not find the Operation corresponding to'
' id={u}'.format(u=op_uuid))
raise Exception('Encountered issue when trying update an Operation'
' database instance after ingesting from repository.')
try:
perform_operation_ingestion(repository_url, operation_uuid_str,
commit_id)
except Exception:
operation.successful_ingestion = False
operation.save()
def test_operation_rejected_for_validation(
self, mock_shutil, mock_read_operation_json, mock_check_for_repo,
mock_checkout_branch, mock_clone_repository,
mock_retrieve_commit_hash, mock_save_operation,
mock_check_required_files, mock_retrieve_repo_name,
mock_prepare_operation):
'''
Here, leave out the workspace_operation key and check
that it will raise an error.
'''
j = json.load(open(os.path.join(TESTDIR, 'invalid_operation.json')))
mock_read_operation_json.return_value = j
mock_hash = 'abcd'
mock_dir = '/some/mock/staging/dir'
mock_clone_repository.return_value = mock_dir
mock_retrieve_commit_hash.return_value = 'abcd'
repo_url = 'http://github.com/some-repo/'
repo_name = 'some-repo'
mock_retrieve_repo_name.return_value = repo_name
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
o = OperationDbModel.objects.create(id=str(op_uuid))
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
with self.assertRaises(ValidationError):
perform_operation_ingestion(repo_url, str(op_uuid), None)
mock_clone_repository.assert_called_with(repo_url)
mock_retrieve_commit_hash.assert_called_with(mock_dir)
mock_save_operation.assert_not_called()
mock_shutil.rmtree.assert_called_with(mock_dir)
mock_checkout_branch.assert_not_called()
mock_check_for_repo.assert_called_with(repo_url)
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 0)
def test_operation_overwrite_blocked(
self, mock_settings, mock_shutil, mock_read_operation_json,
mock_check_for_repo, mock_checkout_branch, mock_clone_repository,
mock_retrieve_commit_hash, mock_check_required_files,
mock_retrieve_repo_name, mock_prepare_operation):
'''
If we do not specifically request that an operation be overwritten,
we check that the operation is appropriately marked as failed.
'''
# make a tmp directory which will act as an existing operation library dir
mock_ops_dir = '/tmp/some-fake-op-dir'
os.mkdir(mock_ops_dir)
mock_settings.OPERATION_LIBRARY_DIR = mock_ops_dir
mock_settings.OPERATION_SPEC_FILENAME = 'foo'
# setup the remainder of the mocks
mock_read_operation_json.return_value = self.valid_dict
mock_hash = 'abcd'
mock_dir = '/some/mock/staging/dir'
mock_clone_repository.return_value = mock_dir
mock_retrieve_commit_hash.return_value = 'abcd'
repo_url = 'http://github.com/some-repo/'
repo_name = 'some-repo'
mock_retrieve_repo_name.return_value = repo_name
n0 = len(OperationDbModel.objects.all())
op_uuid = uuid.uuid4()
# pretend we had an active model prior to this attempted (but failed)
# ingestion process
o = OperationDbModel.objects.create(id=str(op_uuid),
active=True,
successful_ingestion=True)
n1 = len(OperationDbModel.objects.all())
n2 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n1 - n0, 1)
# create the mock operation dir
op_dir_path = os.path.join(mock_ops_dir, str(op_uuid))
os.mkdir(op_dir_path)
with self.assertRaises(Exception):
perform_operation_ingestion(repo_url, str(op_uuid), None)
mock_clone_repository.assert_called_with(repo_url)
mock_retrieve_commit_hash.assert_called_with(mock_dir)
mock_shutil.rmtree.assert_called_with(mock_dir)
mock_checkout_branch.assert_not_called()
mock_check_for_repo.assert_called_with(repo_url)
# check that we did NOT add another operation
n3 = len(OperationDbModel.objects.filter(active=True))
self.assertEqual(n3 - n2, 0)
# If the ingestion failed due to the failure to overwrite,
# we want to maintain the former operation with that same UUID
op = OperationDbModel.objects.get(id=op_uuid)
self.assertTrue(op.successful_ingestion)
self.assertTrue(op.active)
# cleanup from this test:
shutil.rmtree(mock_ops_dir)