def get_definition(self):
"""Returns the batch definition represented by this JSON
:returns: The batch definition
:rtype: :class:`batch.definition.definition.BatchDefinition`:
"""
definition = BatchDefinition()
if 'dataset' in self._definition:
definition.dataset = self._definition['dataset']
if 'forced_nodes' in self._definition:
definition.forced_nodes = ForcedNodesV6(
self._definition['forced_nodes']).get_forced_nodes()
if 'supersedes' in self._definition:
definition.supersedes = self._definition['supersedes']
if 'previous_batch' in self._definition:
prev_batch_dict = self._definition['previous_batch']
definition.root_batch_id = prev_batch_dict['root_batch_id']
if 'forced_nodes' in prev_batch_dict:
definition.forced_nodes = ForcedNodesV6(
prev_batch_dict['forced_nodes']).get_forced_nodes()
return definition
def __init__(self, definition=None, do_validate=False):
"""Creates a v6 batch definition JSON object from the given dictionary
:param definition: The batch definition JSON dict
:type definition: dict
:param do_validate: Whether to perform validation on the JSON schema
:type do_validate: bool
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the given definition is invalid
"""
if not definition:
definition = {}
self._definition = definition
if 'version' not in self._definition:
self._definition['version'] = SCHEMA_VERSION
if self._definition['version'] not in SCHEMA_VERSIONS:
msg = '%s is an unsupported version number' % self._definition[
'version']
raise InvalidDefinition('INVALID_BATCH_DEFINITION', msg)
try:
if do_validate:
validate(self._definition, BATCH_DEFINITION_SCHEMA)
if 'forced_nodes' in self._definition:
ForcedNodesV6(self._definition['forced_nodes'],
do_validate=True)
except ValidationError as ex:
raise InvalidDefinition(
'INVALID_BATCH_DEFINITION',
'Invalid batch definition: %s' % unicode(ex))
def from_json(json_dict):
"""See :meth:`messaging.messages.message.CommandMessage.from_json`
"""
message = CreateRecipes()
message.event_id = json_dict['event_id']
message.create_recipes_type = json_dict['create_recipes_type']
if 'batch_id' in json_dict:
message.batch_id = json_dict['batch_id']
if 'forced_nodes' in json_dict:
message.forced_nodes = ForcedNodesV6(
json_dict['forced_nodes']).get_forced_nodes()
if message.create_recipes_type == REPROCESS_TYPE:
message.recipe_type_name = json_dict['recipe_type_name']
message.recipe_type_rev_num = json_dict['recipe_type_rev_num']
message.root_recipe_ids = json_dict['root_recipe_ids']
elif message.create_recipes_type == SUB_RECIPE_TYPE:
message.recipe_id = json_dict['recipe_id']
if 'root_recipe_id' in json_dict:
message.root_recipe_id = json_dict['root_recipe_id']
if 'superseded_recipe_id' in json_dict:
message.superseded_recipe_id = json_dict[
'superseded_recipe_id']
for sub_dict in json_dict['sub_recipes']:
sub_recipe = SubRecipe(sub_dict['recipe_type_name'],
sub_dict['recipe_type_rev_num'],
sub_dict['node_name'],
sub_dict['process_input'])
message.add_subrecipe(sub_recipe)
return message
def test_invalid_additional_property(self):
"""Tests calling ForcedNodesV6() with json that doesn't match the schema"""
json_data = {'all': False, 'invalid': 'test'}
#no error without validate call, raises error when validating
ForcedNodesV6(forced_nodes=json_data, do_validate=False)
self.assertRaises(InvalidDiff, ForcedNodesV6, json_data, True)
def test_missing_all(self):
"""Tests calling ForcedNodesV6() with json that doesn't match the schema"""
json_data = {'nodes': ['missing-all']}
#no error without validate call, raises error when validating
ForcedNodesV6(forced_nodes=json_data, do_validate=False)
self.assertRaises(InvalidDiff, ForcedNodesV6, json_data, True)
def from_json(json_dict):
"""See :meth:`messaging.messages.message.CommandMessage.from_json`
"""
message = ProcessRecipeInput()
message.recipe_id = json_dict['recipe_id']
if 'forced_nodes' in json_dict:
message.forced_nodes = ForcedNodesV6(json_dict['forced_nodes']).get_forced_nodes()
return message
def _post_v6(self, request, recipe_id):
"""Schedules a recipe for reprocessing and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param recipe_id: The id of the recipe
:type recipe_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
forced_nodes_json = rest_util.parse_dict(request, 'forced_nodes', required=True)
revision_num = rest_util.parse_dict(request, 'revision_num', required=False)
try:
forced_nodes = ForcedNodesV6(forced_nodes_json, do_validate=True)
except InvalidDiff as ex:
logger.exception('Unable to reprocess recipe. Invalid input: %s', forced_nodes_json)
raise BadParameter(unicode(ex))
try:
recipe = Recipe.objects.select_related('recipe_type').get(id=recipe_id)
if revision_num:
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, revision_num)
else:
revision_num = recipe.recipe_type.revision_num
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, recipe.recipe_type.revision_num)
except Recipe.DoesNotExist:
raise Http404
except RecipeTypeRevision.DoesNotExist:
raise Http404
if recipe.is_superseded:
raise BadParameter('Cannot reprocess a superseded recipe')
validation = recipe.recipe_type_rev.validate_forced_nodes(forced_nodes_json)
if not validation.is_valid:
raise BadParameter('Unable to reprocess recipe. Errors in validating forced_nodes: %s' % validation.errors)
if validation.warnings:
logger.warning('Warnings encountered when reprocessing: %s' % validation.warnings)
event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now())
root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id
recipe_type_name = recipe.recipe_type.name
# Execute all of the messages to perform the reprocess
messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id,
forced_nodes=forced_nodes.get_forced_nodes())
CommandMessageManager().send_messages(messages)
return Response(status=status.HTTP_202_ACCEPTED)
def test_full_json(self):
"""Tests calling ForcedNodesV6() with full valid json"""
json_data = {
'version': '6',
'all': False,
'nodes': ['job_a_1', 'job_a_2', 'recipe_b', 'recipe_c'],
'sub_recipes': {
'recipe_b': {
'all': True
},
'recipe_c': {
'all': False,
'nodes': ['job_c_1', 'job_c_2']
}
}
}
#no error without validate call, raises error when validating
fn = ForcedNodesV6(forced_nodes=json_data, do_validate=True)
self.assertDictEqual(fn.get_dict(), json_data)
def test_minimal(self):
"""Tests calling ForcedNodesV6() with no args"""
min = ForcedNodesV6()
self.assertDictEqual(min.get_dict(), {'version': '7', 'all': False})
