def post(self, request):
"""Validates a new Strike process and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
configuration = rest_util.parse_dict(request, 'configuration')
rest_util.parse_string(request, 'title', required=False)
rest_util.parse_string(request, 'description', required=False)
# Validate the Strike configuration
try:
config = StrikeConfiguration(configuration)
warnings = config.validate()
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to validate new Strike process: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def edit_strike(self, strike_id, title=None, description=None, configuration=None):
"""Edits the given Strike process and saves the changes in the database. All database changes occur in an atomic
transaction. An argument of None for a field indicates that the field should not change.
:param strike_id: The unique identifier of the Strike process to edit
:type strike_id: int
:param title: The human-readable name of this Strike process
:type title: string
:param description: A description of this Strike process
:type description: string
:param configuration: The Strike process configuration
:type configuration: dict
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
strike = Strike.objects.get(pk=strike_id)
# Validate the configuration, no exception is success
if configuration:
config = StrikeConfiguration(configuration)
config.validate()
strike.configuration = config.get_dict()
# Update editable fields
if title:
strike.title = title
if description:
strike.description = description
strike.save()
def post(self, request):
"""Validates a new Strike process and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
configuration = rest_util.parse_dict(request, 'configuration')
rest_util.parse_string(request, 'title', required=False)
rest_util.parse_string(request, 'description', required=False)
# Validate the Strike configuration
try:
config = StrikeConfiguration(configuration)
warnings = config.validate()
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to validate new Strike process: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def edit_strike(self,
strike_id,
title=None,
description=None,
configuration=None):
"""Edits the given Strike process and saves the changes in the database. All database changes occur in an atomic
transaction. An argument of None for a field indicates that the field should not change.
:param strike_id: The unique identifier of the Strike process to edit
:type strike_id: int
:param title: The human-readable name of this Strike process
:type title: string
:param description: A description of this Strike process
:type description: string
:param configuration: The Strike process configuration
:type configuration: dict
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
strike = Strike.objects.get(pk=strike_id)
# Validate the configuration, no exception is success
if configuration:
config = StrikeConfiguration(configuration)
config.validate()
strike.configuration = config.get_dict()
# Update editable fields
if title:
strike.title = title
if description:
strike.description = description
strike.save()
def get_configuration(self):
"""Returns the strike configuration represented by this JSON
:returns: The strike configuration
:rtype: :class:`ingest.strike.configuration.strike_configuration.StrikeConfiguration`:
"""
config = StrikeConfiguration()
config.configuration = self._configuration
config.file_handler = self._file_handler
return config
def test_conversion_from_1_0(self):
"""Tests calling StrikeConfiguration.validate() after converting from schema version 1.0"""
old_config = {
'version':
'1.0',
'transfer_suffix':
'_tmp',
'mount':
'host:/my/path',
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.new_workspace.name,
}],
}
strike_config = StrikeConfiguration(old_config)
strike_config.validate()
auto_workspace = Workspace.objects.get(name__contains='auto')
new_config = {
'version':
'2.0',
'workspace':
auto_workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp'
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.new_workspace.name,
}]
}
self.assertDictEqual(strike_config._configuration, new_config)
auto_workspace_config = {
'version': '1.0',
'broker': {
'type': 'host',
'host_path': '/my/path'
}
}
self.assertDictEqual(auto_workspace.json_config, auto_workspace_config)
def validate_strike(self, name, configuration):
"""Validates a new Strike process prior to attempting a save
:param name: The identifying name of a Strike process to validate
:type name: string
:param configuration: The Strike process configuration
:type configuration: dict
:returns: A list of warnings discovered during validation.
:rtype: list[:class:`ingest.strike.configuration.strike_configuration.ValidationWarning`]
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
warnings = []
# Validate the configuration, no exception is success
StrikeConfiguration(configuration)
# Check for issues when changing an existing Strike configuration
try:
strike = Strike.objects.get(name=name)
if (configuration['mount'] and strike.configuration['mount'] and
configuration['mount'] != strike.configuration['mount']):
warnings.append(
ValidationWarning(
'mount_change',
'Changing the mount path may disrupt file monitoring.')
)
except Strike.DoesNotExist:
pass
return warnings
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(file_name='my_file.txt')
self.mount = 'host:/path'
self.mount_on = os.path.join('my', 'test')
self.workspace = storage_test_utils.create_workspace()
self.config = StrikeConfiguration({
'version':
'1.0',
'mount':
self.mount,
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': '.*txt',
'workspace_path': 'foo',
'workspace_name': self.workspace.name,
}],
})
self.job_exe = job_test_utils.create_job_exe()
self.strike_proc = StrikeProcessor(1, self.job_exe.id, self.config)
self.strike_dir = SCALE_INGEST_MOUNT_PATH
def test_edit_simple(self):
"""Tests editing only the basic attributes of a Strike process"""
json_data = {
'title': 'Title EDIT',
'description': 'Description EDIT',
}
url = rest_util.get_url('/strikes/%d/' % self.strike.id)
response = self.client.generic('PATCH', url, json.dumps(json_data),
'application/json')
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertTrue(isinstance(result, dict),
'result must be a dictionary')
self.assertEqual(result['id'], self.strike.id)
self.assertEqual(result['title'], 'Title EDIT')
self.assertEqual(result['description'], 'Description EDIT')
self.assertDictEqual(
result['configuration'],
StrikeConfiguration(self.strike.configuration).get_dict())
strike = Strike.objects.get(pk=self.strike.id)
self.assertEqual(strike.title, 'Title EDIT')
self.assertEqual(strike.description, 'Description EDIT')
def test_edit_config(self):
"""Tests editing the configuration of a Strike process"""
config = {
'version': '1.0',
'mount': 'host:/my/path/EDIT',
'transfer_suffix': '_tmp',
'files_to_ingest': [{
'data_types': ['test'],
'filename_regex': '.*txt',
'workspace_path': 'my_path',
'workspace_name': 'raw',
}],
}
url = '/strikes/%d/' % self.strike.id
json_data = {
'configuration': config,
}
response = self.client.generic('PATCH', url, json.dumps(json_data), 'application/json')
result = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
self.assertEqual(result['id'], self.strike.id)
self.assertEqual(result['title'], self.strike.title)
self.assertDictEqual(result['configuration'], StrikeConfiguration(config).get_dict())
strike = Strike.objects.get(pk=self.strike.id)
self.assertEqual(strike.title, self.strike.title)
self.assertDictEqual(strike.configuration, config)
def test_no_match(self):
'''Tests calling StrikeConfiguration.match_file_name() without a match.'''
config = {
'mount': 'host:/my/path',
'transfer_suffix': '_tmp',
'files_to_ingest': [{
'filename_regex': '.*MY.*PATTERN.*',
'data_types': ['one', 'two'],
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.workspace.name,
}],
}
strike_config = StrikeConfiguration(config)
result = strike_config.match_file_name('ababFOOBARabab')
self.assertIsNone(result)
def get_strike_configuration(self):
"""Returns the configuration for this Strike process
:returns: The configuration for this Strike process
:rtype: :class:`ingest.strike.configuration.strike_configuration.StrikeConfiguration`
"""
return StrikeConfiguration(self.configuration)
def get_v5_strike_configuration_as_dict(self):
"""Returns the v5 configuration for this Strike process as a dict
:returns: The configuration for this Strike process
:rtype: dict
"""
return StrikeConfiguration(self.configuration).get_dict()
def test_match(self):
'''Tests calling StrikeConfiguration.match_file_name() with a match.'''
data_types_list = ['one', 'two']
wksp_path = os.path.join('my', 'path')
config = {
'mount': 'host:/my/path',
'transfer_suffix': '_tmp',
'files_to_ingest': [{
'filename_regex': '.*MY.*PATTERN.*',
'data_types': data_types_list,
'workspace_path': wksp_path,
'workspace_name': self.workspace.name,
}],
}
strike_config = StrikeConfiguration(config)
result = strike_config.match_file_name('ababMYgggPATTERNxyz')
self.assertTupleEqual(result, (data_types_list, wksp_path, self.workspace))
def test_conversion_from_1_0(self):
"""Tests calling StrikeConfiguration.validate() after converting from schema version 1.0"""
old_config = {
'version': '1.0',
'transfer_suffix': '_tmp',
'mount': 'host:/my/path',
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.new_workspace.name,
}],
}
strike_config = StrikeConfiguration(old_config)
strike_config.validate()
auto_workspace = Workspace.objects.get(name__contains='auto')
new_config = {
'version': '2.0',
'workspace': auto_workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp'
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.new_workspace.name,
}]
}
self.assertDictEqual(strike_config._configuration, new_config)
auto_workspace_config = {
'version': '1.0',
'broker': {
'type': 'host',
'host_path': '/my/path'
}
}
self.assertDictEqual(auto_workspace.json_config, auto_workspace_config)
def test_no_match(self):
'''Tests calling StrikeConfiguration.match_file_name() without a match.'''
config = {
'mount':
'host:/my/path',
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': '.*MY.*PATTERN.*',
'data_types': ['one', 'two'],
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.workspace.name,
}],
}
strike_config = StrikeConfiguration(config)
result = strike_config.match_file_name('ababFOOBARabab')
self.assertIsNone(result)
def create_strike(self, name, title, description, configuration):
"""Creates a new Strike process with the given configuration and returns the new Strike model. The Strike model
will be saved in the database and the job to run the Strike process will be placed on the queue. All changes to
the database will occur in an atomic transaction.
:param name: The identifying name of this Strike process
:type name: string
:param title: The human-readable name of this Strike process
:type title: string
:param description: A description of this Strike process
:type description: string
:param configuration: The Strike configuration
:type configuration: dict
:returns: The new Strike process
:rtype: :class:`ingest.models.Strike`
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
# Validate the configuration, no exception is success
config = StrikeConfiguration(configuration)
config.validate()
strike = Strike()
strike.name = name
strike.title = title
strike.description = description
strike.configuration = config.get_dict()
strike.save()
strike_type = self.get_strike_job_type()
job_data = JobData()
job_data.add_property_input('Strike ID', unicode(strike.id))
event_description = {'strike_id': strike.id}
event = TriggerEvent.objects.create_trigger_event(
'STRIKE_CREATED', None, event_description, now())
strike.job = Queue.objects.queue_new_job(strike_type, job_data, event)
strike.save()
return strike
def create_strike(self, name, title, description, configuration):
"""Creates a new Strike process with the given configuration and returns the new Strike model. The Strike model
will be saved in the database and the job to run the Strike process will be placed on the queue. All changes to
the database will occur in an atomic transaction.
:param name: The identifying name of this Strike process
:type name: string
:param title: The human-readable name of this Strike process
:type title: string
:param description: A description of this Strike process
:type description: string
:param configuration: The Strike configuration
:type configuration: dict
:returns: The new Strike process
:rtype: :class:`ingest.models.Strike`
:raises :class:`ingest.strike.configuration.exceptions.InvalidStrikeConfiguration`: If the configuration is
invalid.
"""
# Validate the configuration, no exception is success
config = StrikeConfiguration(configuration)
config.validate()
strike = Strike()
strike.name = name
strike.title = title
strike.description = description
strike.configuration = config.get_dict()
strike.save()
strike_type = self.get_strike_job_type()
job_data = JobData()
job_data.add_property_input('Strike ID', unicode(strike.id))
event_description = {'strike_id': strike.id}
event = TriggerEvent.objects.create_trigger_event('STRIKE_CREATED', None, event_description, now())
strike.job = Queue.objects.queue_new_job(strike_type, job_data, event)
strike.save()
return strike
def test_match(self):
'''Tests calling StrikeConfiguration.match_file_name() with a match.'''
data_types_list = ['one', 'two']
wksp_path = os.path.join('my', 'path')
config = {
'mount':
'host:/my/path',
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': '.*MY.*PATTERN.*',
'data_types': data_types_list,
'workspace_path': wksp_path,
'workspace_name': self.workspace.name,
}],
}
strike_config = StrikeConfiguration(config)
result = strike_config.match_file_name('ababMYgggPATTERNxyz')
self.assertTupleEqual(result,
(data_types_list, wksp_path, self.workspace))
def test_bare_min(self):
"""Tests calling StrikeConfiguration constructor with bare minimum JSON"""
# No exception is success
StrikeConfiguration({
'workspace': self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt'
}],
})
def test_validate_bad_monitor_type(self):
"""Tests calling StrikeConfiguration.validate() with a bad monitor type"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 'BAD',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
}],
}
self.assertRaises(InvalidStrikeConfiguration, StrikeConfiguration(config).validate)
def test_validate_mismatched_monitor_type(self):
"""Tests calling StrikeConfiguration.validate() with a monitor type that does not match the broker type"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 's3',
'sqs_name': 'my-sqs',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
}],
}
self.assertRaises(InvalidStrikeConfiguration, StrikeConfiguration(config).validate)
def test_bare_min(self):
'''Tests calling StrikeConfiguration constructor with bare minimum JSON.'''
# No exception is success
StrikeConfiguration({
'mount':
'host:/my/path',
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': 'hello',
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.workspace.name,
}],
})
def test_validate_workspace_not_active(self):
"""Tests calling StrikeConfiguration.validate() with a new workspace that is not active"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'new_workspace': self.inactive_workspace.name,
}],
}
self.assertRaises(InvalidStrikeConfiguration, StrikeConfiguration(config).validate)
def test_successful(self):
'''Tests calling StrikeConfiguration constructor successfully with all information.'''
config = {
'mount':
'host:/my/path',
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': 'hello',
'data_types': ['one', 'two'],
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.workspace.name,
}],
}
# No exception is success
StrikeConfiguration(config)
def test_successful_all(self):
"""Tests calling StrikeConfiguration constructor successfully with all information"""
config = {
'workspace': self.workspace.name,
'monitor': {
'type': 'dir-watcher',
'transfer_suffix': '_tmp',
},
'files_to_ingest': [{
'filename_regex': '.*txt',
'data_types': ['one', 'two'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.workspace.name,
}],
}
# No exception is success
StrikeConfiguration(config)
def create_strike_process(self, name, title, description, configuration):
'''Creates a new Strike process with the given configuration and returns the new Strike model. The Strike model
will be saved in the database and the job to run the Strike process will be placed on the queue. All changes to
the database will occur in an atomic transaction.
:param name: The stable name of this Strike process
:type name: str
:param title: The human-readable name of this Strike process
:type title: str
:param description: A description of this Strike process
:type description: str
:param configuration: The Strike configuration
:type configuration: dict
:returns: The new Strike process
:rtype: :class:`ingest.models.Strike`
:raises InvalidStrikeConfiguration: If the configuration is invalid
'''
# Validate the configuration, no exception is success
StrikeConfiguration(configuration)
strike = Strike()
strike.name = name
strike.title = title
strike.description = description
strike.configuration = configuration
strike.save()
strike_type = self.get_strike_job_type()
job_data = {
'input_data': [{
'name': 'Strike ID',
'value': unicode(strike.id)
}]
}
event_description = {'strike_id': strike.id}
event = TriggerEvent.objects.create_trigger_event(
'STRIKE_CREATED', None, event_description, now())
job_id = Queue.objects.queue_new_job(strike_type, job_data, event)
strike.job_id = job_id
strike.save()
return strike
def test_successful(self):
'''Tests calling StrikeConfiguration() successfully.'''
config = {
'mount':
'host:/my/path',
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': 'hello',
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.workspace_1.name,
}, {
'filename_regex': 'hello',
'workspace_path': os.path.join('my', 'path'),
'workspace_name': self.workspace_2.name,
}],
}
# No exception is success
StrikeConfiguration(config)
