def test_update_strike(self, mock_msg_mgr):
"""Tests calling StrikeManager.create_strike successfully with v6 config"""
config = {
'version':
'6',
'workspace':
self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp'
},
'files_to_ingest': [{
'filename_regex': 'foo',
'data_types': ['test1', 'test2'],
'new_workspace': self.workspace.name,
'new_file_path': 'my/path'
}],
'recipe': {
'name': self.recipe.name,
'revision_num': self.recipe.revision_num
},
}
config = StrikeConfigurationV6(config).get_configuration()
strike = Strike.objects.create_strike('my_name', 'my_title',
'my_description', config)
self.assertEqual(strike.job.status, 'QUEUED')
config = {
'version':
'6',
'workspace':
self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp'
},
'files_to_ingest': [{
'filename_regex': 'foo',
'data_types': ['test1', 'test2'],
'new_workspace': self.workspace.name,
'new_file_path': 'my/new_path'
}],
'recipe': {
'name': self.recipe.name,
'revision_num': self.recipe.revision_num
},
}
config = StrikeConfigurationV6(config).get_configuration()
Strike.objects.edit_strike(strike.id, 'my_title2', 'my_description2',
config)
strike = Strike.objects.get_details(strike.id)
self.assertEqual(strike.title, 'my_title2')
self.assertEqual(strike.description, 'my_description2')
self.assertEqual(
strike.configuration['files_to_ingest'][0]['new_file_path'],
'my/new_path')
def test_validate_successful_all(self):
"""Tests calling StrikeConfiguration.validate() successfully with all information"""
recipe = recipe_test_utils.create_recipe_type_v6(
definition=recipe_test_utils.RECIPE_DEFINITION)
config = {
'workspace':
self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.new_workspace.name,
}],
'recipe': {
'name': recipe.name,
'revision_num': recipe.revision_num,
},
}
# No exception is success
configuration = StrikeConfigurationV6(config).get_configuration()
configuration.validate()
def create_impl_v6(self, request):
"""Creates a new Strike process and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=True)
name = title_to_name(self.queryset, title)
description = rest_util.parse_string(request, 'description', required=False)
configuration = rest_util.parse_dict(request, 'configuration')
config = None
try:
if configuration:
config = StrikeConfigurationV6(configuration, do_validate=True).get_configuration()
except InvalidStrikeConfiguration as ex:
raise BadParameter('Strike configuration invalid: %s' % unicode(ex))
try:
strike = Strike.objects.create_strike(name, title, description, config)
except InvalidStrikeConfiguration as ex:
raise BadParameter('Strike configuration invalid: %s' % unicode(ex))
# Fetch the full strike process with details
try:
strike = Strike.objects.get_details(strike.id)
except Strike.DoesNotExist:
raise Http404
serializer = StrikeDetailsSerializerV6(strike)
strike_url = reverse('strike_details_view', args=[strike.id], request=request)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=strike_url))
def edit_strike_v6(self, strike_id, title=None, description=None, configuration=None):
"""Edits the given Strike process and saves the changes in the database. All database changes occur in an atomic
transaction. An argument of None for a field indicates that the field should not change.
:param strike_id: The unique identifier of the Strike process to edit
:type strike_id: int
:param title: The human-readable name of this Strike process
:type title: string
:param description: A description of this Strike process
:type description: string
:param configuration: The Strike process configuration
:type configuration: dict
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
strike = Strike.objects.get(pk=strike_id)
# Validate the configuration, no exception is success
if configuration:
config = StrikeConfigurationV6(configuration)
config.validate()
strike.configuration = config.get_dict()
# Update editable fields
if title:
strike.title = title
if description:
strike.description = description
strike.save()
def get_strike_configuration(self):
"""Returns the configuration for this Strike process
:returns: The configuration for this Strike process
:rtype: :class:`ingest.strike.configuration.strike_configuration.StrikeConfiguration`
"""
return StrikeConfigurationV6(self.configuration).get_configuration()
def test_bare_min(self):
"""Tests calling StrikeConfigurationV6 constructor with bare minimum JSON"""
# No exception is success
StrikeConfigurationV6({
'workspace': self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt'
}],
}, do_validate=True)
def test_conversion_from_1_0(self):
"""Tests calling StrikeConfigurationV6.validate() after converting from schema version 1.0"""
old_config = {
'version':
'1.0',
'transfer_suffix':
'_tmp',
'mount':
'host:/my/path',
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.new_workspace.name,
}],
}
strike_config = StrikeConfigurationV6(old_config)
strike_config.validate()
auto_workspace = Workspace.objects.get(name__contains='auto')
new_config = {
'version':
'6',
'workspace':
auto_workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp'
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.new_workspace.name,
}]
}
self.assertDictEqual(strike_config._configuration, new_config)
auto_workspace_config = {
'version': '1.0',
'broker': {
'type': 'host',
'host_path': '/my/path'
}
}
self.assertDictEqual(auto_workspace.json_config, auto_workspace_config)
def test_validate_mismatched_monitor_type(self):
"""Tests calling StrikeConfigurationV6.validate() with a monitor type that does not match the broker type"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 's3',
'sqs_name': 'my-sqs',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
}],
}
configuration = StrikeConfigurationV6(config).get_configuration()
self.assertRaises(InvalidStrikeConfiguration, configuration.validate)
def test_validate_bad_monitor_type(self):
"""Tests calling StrikeConfigurationV6.validate() with a bad monitor type"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 'BAD',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
}],
}
configuration = StrikeConfigurationV6(config).get_configuration()
self.assertRaises(InvalidStrikeConfiguration, configuration.validate)
def patch_impl_v6(self, request, strike_id):
"""Edits an existing Strike process and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param strike_id: The ID of the Strike process
:type strike_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
configuration = rest_util.parse_dict(request, 'configuration', required=False)
config = None
try:
if configuration:
config = StrikeConfigurationV6(configuration, do_validate=True).get_configuration()
except InvalidStrikeConfiguration as ex:
raise BadParameter('Strike configuration invalid: %s' % unicode(ex))
try:
# must collect old config before editing strike
if config:
new_config = config.get_dict()
old_config = Strike.objects.get_details(strike_id)
Strike.objects.edit_strike(strike_id, title, description, config)
# if workspace has changed strike job must be restarted for changes to take effect
if config and old_config.configuration["workspace"] != new_config["workspace"]:
strike_job = old_config.job
Job.objects.update_jobs_to_canceled([strike_job], timezone.now())
requeue_jobs = []
requeue_jobs.append(QueuedJob(strike_job.id, strike_job.num_exes))
msg = create_requeue_jobs_messages(requeue_jobs)
CommandMessageManager().send_messages(msg)
except Strike.DoesNotExist:
raise Http404
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to edit Strike process: %s', strike_id)
raise BadParameter(unicode(ex))
return Response(status=status.HTTP_204_NO_CONTENT)
def test_validate_workspace_not_active(self):
"""Tests calling StrikeConfigurationV6.validate() with a new workspace that is not active"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'new_workspace': self.inactive_workspace.name,
}],
}
configuration = StrikeConfigurationV6(config).get_configuration()
self.assertRaises(InvalidStrikeConfiguration, configuration.validate)
def test_successful_all(self):
"""Tests calling StrikeConfigurationV6 constructor successfully with all information"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.workspace.name,
}],
}
# No exception is success
StrikeConfigurationV6(config)
def test_validate_bad_workspace(self):
"""Tests calling StrikeConfigurationV6.validate() with a bad workspace"""
config = {
'workspace':
self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'new_workspace': 'BADWORKSPACE',
}],
}
self.assertRaises(InvalidStrikeConfiguration,
StrikeConfigurationV6(config).validate)
def validate_strike_v6(self, configuration):
"""Validates the given configuration for creating a new strike process
:param configuration: The strike configuration
:type configuration: dict
:returns: The strike validation
:rtype: :class:`strike.models.StrikeValidation`
"""
is_valid = True
errors = []
warnings = []
try:
config = StrikeConfigurationV6(configuration)
warnings = config.validate()
except InvalidStrikeConfiguration as ex:
is_valid = False
errors.append(ex.error)
return StrikeValidation(is_valid, errors, warnings)
def create_strike_v6(self, name, title, description, configuration):
"""Creates a new Strike process with the given configuration and returns the new Strike model. The Strike model
will be saved in the database and the job to run the Strike process will be placed on the queue. All changes to
the database will occur in an atomic transaction.
:param name: The identifying name of this Strike process
:type name: string
:param title: The human-readable name of this Strike process
:type title: string
:param description: A description of this Strike process
:type description: string
:param configuration: The Strike configuration
:type configuration: dict
:returns: The new Strike process
:rtype: :class:`ingest.models.Strike`
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
# Validate the configuration, no exception is success
config = StrikeConfigurationV6(configuration)
config.validate()
strike = Strike()
strike.name = name
strike.title = title
strike.description = description
strike.configuration = config.get_dict()
strike.save()
strike_type = self.get_strike_job_type()
job_data = JobData()
job_data.add_property_input('Strike ID', unicode(strike.id))
event_description = {'strike_id': strike.id}
event = TriggerEvent.objects.create_trigger_event('STRIKE_CREATED', None, event_description, now())
strike.job = Queue.objects.queue_new_job(strike_type, job_data, event)
strike.save()
return strike
def patch_impl_v6(self, request, strike_id):
"""Edits an existing Strike process and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param strike_id: The ID of the Strike process
:type strike_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request,
'description',
required=False)
configuration = rest_util.parse_dict(request,
'configuration',
required=False)
config = None
try:
if configuration:
config = StrikeConfigurationV6(
configuration, do_validate=True).get_configuration()
except InvalidStrikeConfiguration as ex:
raise BadParameter('Strike configuration invalid: %s' %
unicode(ex))
try:
Strike.objects.edit_strike(strike_id, title, description, config)
except Strike.DoesNotExist:
raise Http404
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to edit Strike process: %s', strike_id)
raise BadParameter(unicode(ex))
return Response(status=status.HTTP_204_NO_CONTENT)
def test_successful_v6(self):
"""Tests calling StrikeManager.create_strike successfully with v6 config"""
config = {
'version':
'6',
'workspace':
self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp'
},
'files_to_ingest': [{
'filename_regex': 'foo',
'data_types': ['test1', 'test2'],
'new_workspace': self.workspace.name,
'new_file_path': 'my/path'
}]
}
config = StrikeConfigurationV6(config).get_configuration()
strike = Strike.objects.create_strike('my_name', 'my_title',
'my_description', config)
self.assertEqual(strike.job.status, 'QUEUED')
def test_successful_recipe_kickoff(self, mock_msg_mgr, mock_msg_mgr_rc, mock_msg_mgr_q):
"""Tests successfully producing an ingest that immediately calls a recipe"""
strike_config = {
'version': '7',
'workspace': self.workspace.name,
'monitor': {'type': 'dir-watcher', 'transfer_suffix': '_tmp'},
'files_to_ingest': [{
'filename_regex': 'input_file',
'data_types': ['image_type'],
'new_workspace': self.workspace.name,
'new_file_path': 'my/path'
}],
'recipe': {
'name': self.recipe_v7.name
},
}
config = StrikeConfigurationV6(strike_config).get_configuration()
strike = Strike.objects.create_strike('my_name', 'my_title', 'my_description', config)
ingest = ingest_test_utils.create_ingest(source_file=self.source_file)
# Call method to test
IngestRecipeHandler().process_ingested_source_file(ingest.id, strike, self.source_file, now())
self.assertEqual(Recipe.objects.count(), 1)
self.assertEqual(Recipe.objects.first().recipe_type.name, self.recipe_v7.name)
# Verify ingest event and trigger event objects were created
from ingest.models import IngestEvent
events = IngestEvent.objects.all().values()
self.assertEqual(len(events), 1)
self.assertEqual(events[0]['type'], 'STRIKE')
# Create scan
scan_config = {
'workspace': self.workspace.name,
'scanner': {
'type': 'dir'
},
'files_to_ingest': [{
'filename_regex': 'input_file',
'data_types': ['type1'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.workspace.name,
}],
'recipe': {
'name': self.recipe_v7.name,
},
}
scan_configuration = ScanConfigurationV6(scan_config).get_configuration()
scan = Scan.objects.create_scan('my_name', 'my_title', 'my_description', scan_configuration)
# Call method to test
IngestRecipeHandler().process_ingested_source_file(ingest.id, scan, self.source_file, now())
self.assertEqual(Recipe.objects.count(), 2)
self.assertEqual(Recipe.objects.last().recipe_type.name, self.recipe_v7.name)
# Verify events were created
events = IngestEvent.objects.all().values()
self.assertEqual(len(events), 2)
self.assertEqual(events[1]['type'], 'SCAN')
# Update the recipe then call ingest with revision 1
manifest = job_test_utils.create_seed_manifest(
inputs_files=[{'name': 'INPUT_FILE', 'media_types': ['text/plain'], 'required': True, 'multiple': True}], inputs_json=[])
jt2 = job_test_utils.create_seed_job_type(manifest=manifest)
definition = {'version': '7',
'input': {'files': [{'name': 'INPUT_FILE',
'media_types': ['text/plain'],
'required': True,
'multiple': True}],
'json': []},
'nodes': {'node_a': {'dependencies': [],
'input': {'INPUT_FILE': {'type': 'recipe', 'input': 'INPUT_FILE'}},
'node_type': {'node_type': 'job', 'job_type_name': self.jt1.name,
'job_type_version': self.jt1.version,
'job_type_revision': 1}},
'node_b': {'dependencies': [],
'input': {'INPUT_FILE': {'type': 'recipe', 'input': 'INPUT_FILE'}},
'node_type': {'node_type': 'job', 'job_type_name': jt2.name,
'job_type_version': jt2.version,
'job_type_revision': 1}}}}
recipe_test_utils.edit_recipe_type_v6(recipe_type=self.recipe, definition=definition)
strike_config['recipe'] = {
'name': self.recipe.name,
'revision_num': 1,
}
config = StrikeConfigurationV6(strike_config).get_configuration()
strike = Strike.objects.create_strike('my_name_2', 'my_title_2', 'my_description_2', config)
ingest = ingest_test_utils.create_ingest(source_file=self.source_file)
# Call method to test
IngestRecipeHandler().process_ingested_source_file(ingest.id, strike, self.source_file, now())
self.assertEqual(Recipe.objects.count(), 3)
self.assertEqual(Recipe.objects.first().recipe_type.name, self.recipe.name)
# Verify events were created
events = IngestEvent.objects.all().values()
self.assertEqual(len(events), 3)
self.assertEqual(events[2]['type'], 'STRIKE')
