def _generate_msg_queue_status(self):
"""Generates the Message Queue status message
AMQP (rabbitmq)
"""
status_dict = {'OK': False, 'detail': {}, 'errors': [], 'warnings': []}
status_dict['detail']['msg'] = 'Message Queue is healthy'
status_dict['detail']['queue_name'] = scale_settings.QUEUE_NAME
status_dict['detail']['num_message_handlers'] = scheduler_mgr.config.num_message_handlers
status_dict['detail']['queue_depth'] = 0
status_dict['detail']['region_name'] = ''
status_dict['detail']['type'] = ''
try:
broker_details = BrokerDetails.from_broker_url(scale_settings.BROKER_URL)
except InvalidBrokerUrl:
msg = 'Error parsing broker url'
status_dict['errors'] = [{'INVALID_BROKER_URL': msg}]
status_dict['detail']['msg'] = msg
return status_dict
status_dict['detail']['type'] = broker_details.get_type()
if broker_details.get_type() == 'amqp':
try:
CommandMessageManager().get_queue_size()
status_dict['OK'] = True
except Exception as ex:
logger.error('Error connecting to RabbitMQ: %s' % unicode(ex))
status_dict['OK'] = False
msg = 'Error connecting to RabbitMQ: Check Logs for details'
status_dict['errors'] = [{'RABBITMQ_ERROR': msg}]
status_dict['detail']['msg'] = 'Unable to get message queue size.'
elif broker_details.get_type() == 'sqs':
status_dict['detail']['region_name'] = broker_details.get_address()
try:
CommandMessageManager().get_queue_size()
status_dict['OK'] = True
except Exception as ex:
logger.error('Unable to get queue size from sqs: %s' % unicode(ex))
msg = 'Error connecting to SQS: Check Logs for details'
status_dict['OK'] = False
status_dict['errors'] = [{'SQS_ERROR': msg}]
status_dict['detail']['msg'] = 'Unable to get message queue size.'
else:
status_dict['OK'] = False
status_dict['detail']['msg'] = 'Broker is an unsupported type: %s' % broker_details.get_type()
if status_dict['OK']:
status_dict['detail']['queue_depth'] = CommandMessageManager().get_queue_size()
if scale_settings.MESSSAGE_QUEUE_DEPTH_WARN > 0 and status_dict['detail']['queue_depth'] > scale_settings.MESSSAGE_QUEUE_DEPTH_WARN:
status_dict['warnings'].append({'LARGE_QUEUE': 'Message queue is very large'})
status_dict['detail']['msg'] = 'Message queue is large. Scale may be unresponsive.'
return status_dict
def process_event(self, event, last_event=None):
"""See :meth:`job.clock.ClockEventProcessor.process_event`.
Compares the new event with the last event any missing metrics jobs.
"""
# Attempt to get the daily metrics job type
try:
job_type = JobType.objects.filter(
name='scale-daily-metrics').last()
except JobType.DoesNotExist:
raise ClockEventError(
'Missing required job type: scale-daily-metrics')
if last_event:
# Build a list of days that require metrics
day_count = xrange(
(event.occurred.date() - last_event.occurred.date()).days)
days = [
last_event.occurred.date() + datetime.timedelta(days=d)
for d in day_count
]
else:
# Use the previous day when first triggered
days = [timezone.now().date() - datetime.timedelta(days=1)]
# Schedule one job for each required day
for day in days:
job_data = Data()
job_data.add_value(JsonValue('DAY', day.strftime('%Y-%m-%d')))
job = Queue.objects.queue_new_job_v6(job_type, job_data, event)
CommandMessageManager().send_messages(
create_process_job_input_messages([job.id]))
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the command.
"""
count = options.get('count')
if not count:
count = 1
logger.info(
'Command starting: scale_echo_message - sending {} message(s)'.
format(count))
manager = CommandMessageManager()
messages = []
for x in range(count):
messages.append(
EchoCommandMessage.from_json({
'message':
'Greetings, this is echo #{} at {}!'.format(
x + 1, datetime.utcnow())
}))
manager.send_messages(messages)
logger.info('Command completed: scale_echo_message')
def process_manual_ingested_source_file(self, ingest_id, source_file, when,
recipe_type_id):
"""Processes a manual ingest where a strike or scan is not involved. All database
changes are made in an atomic transaction
:param ingest_id:
:type ingest_id: int
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
:param recipe_type_id: id of the Recipe type to kick off
:type recipe_type_id: int
"""
recipe_type = RecipeType.objects.get(id=recipe_type_id)
if recipe_type and recipe_type.is_active:
recipe_data = Data()
input_name = recipe_type.get_definition().get_input_keys()[0]
recipe_data.add_value(FileValue(input_name, [source_file.id]))
event = self._create_trigger_event(None, source_file, when)
ingest_event = self._create_ingest_event(ingest_id, None,
source_file, when)
messages = create_recipes_messages(
recipe_type.name, recipe_type.revision_num,
convert_data_to_v6_json(recipe_data).get_dict(), event.id,
ingest_event.id)
CommandMessageManager().send_messages(messages)
else:
logger.info(
'No recipe type found for id %s or recipe type is inactive' %
recipe_type_id)
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the file destruction process.
"""
# Register a listener to handle clean shutdowns
signal.signal(signal.SIGTERM, self._onsigterm)
files = options.get('files')
workspace_list = options.get('workspace')
purge = options.get('purge')
workspaces = self._detect_workspaces(workspace_list)
files = self._configure_files(files, workspaces)
logger.info('Command starting: scale_delete_files')
logger.info('File IDs: %s', [x.id for x in files])
for wrkspc_name, wrkspc in workspaces.iteritems():
delete_files_job.delete_files(
files=[f for f in files if f.workspace == wrkspc_name],
broker=wrkspc['broker'],
volume_path=wrkspc['volume_path'])
messages = create_delete_files_messages(files=files, purge=purge)
CommandMessageManager().send_messages(messages)
logger.info('Command completed: scale_delete_files')
sys.exit(0)
def test_no_message_send_downstream(self, send_messages):
"""Validate send_message is not called when messages is empty"""
manager = CommandMessageManager()
manager._send_downstream([])
self.assertFalse(send_messages.called)
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the command.
"""
logger.info('Command starting: scale_message_handler')
self.running = True
logger.info('Initializing message handler')
logger.info('Caching builtin errors...')
Error.objects.cache_builtin_errors()
logger.info('Initialization complete, ready to process messages')
# Set the signal handler
signal.signal(signal.SIGINT, self.interupt)
signal.signal(signal.SIGTERM, self.interupt)
manager = CommandMessageManager()
while self.running:
manager.receive_messages()
logger.info('Command completed: scale_message_handler')
def __init__(self):
"""Constructor
"""
super(MessagingThread, self).__init__('Messaging', THROTTLE, WARN_THRESHOLD)
self._manager = CommandMessageManager()
self._messages = []
def _create_v6(self, request):
"""The v6 version for creating batches
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request,
'description',
required=False)
recipe_type_id = rest_util.parse_int(request, 'recipe_type_id')
definition_dict = rest_util.parse_dict(request, 'definition')
configuration_dict = rest_util.parse_dict(request,
'configuration',
required=False)
# Make sure the recipe type exists
try:
recipe_type = RecipeType.objects.get(pk=recipe_type_id)
except RecipeType.DoesNotExist:
raise BadParameter('Unknown recipe type: %d' % recipe_type_id)
# Validate and create the batch
try:
definition = BatchDefinitionV6(definition=definition_dict,
do_validate=True).get_definition()
configuration = BatchConfigurationV6(
configuration=configuration_dict,
do_validate=True).get_configuration()
with transaction.atomic():
event = TriggerEvent.objects.create_trigger_event(
'USER', None, {'user': '******'}, now())
batch = Batch.objects.create_batch_v6(
title,
description,
recipe_type,
event,
definition,
configuration=configuration)
CommandMessageManager().send_messages(
[create_batch_recipes_message(batch.id)])
except InvalidDefinition as ex:
raise BadParameter(unicode(ex))
except InvalidConfiguration as ex:
raise BadParameter(unicode(ex))
# Fetch the full batch with details
batch = Batch.objects.get_details_v6(batch.id)
url = reverse('batch_details_view', args=[batch.id], request=request)
serializer = BatchDetailsSerializerV6(batch)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers={'Location': url})
def test_successful_send_downstream(self, send_messages):
"""Validate call of send_message for each downstream message"""
messages = ['one', 'two']
manager = CommandMessageManager()
manager._send_downstream(messages)
send_messages.assert_called_with(messages)
def test_send_message_no_serializer(self):
"""Validate that send_message raises AttributeError when to_json is not available"""
class InvalidCommand(object):
def __init__(self):
self.type = 'test'
message = InvalidCommand()
manager = CommandMessageManager()
with self.assertRaises(AttributeError):
manager.send_messages([message])
def test_send_messages_no_type(self):
"""Validate that send_message raises AttributeError when message type is not available"""
class InvalidCommand(object):
def to_json(self): # pragma: no cover
pass
message = InvalidCommand()
manager = CommandMessageManager()
with self.assertRaises(AttributeError):
manager.send_messages([message])
def test_receive_message_execute_failure(self, process, broker_details,
get_message_backend):
"""Exercise all exception code paths within receive_message"""
def gen():
yield MagicMock()
process.side_effect = CommandMessageExecuteFailure
manager = CommandMessageManager()
manager._backend = MagicMock()
manager._backend.receive_messages.return_value = gen()
manager.receive_messages()
def _post_v6(self, request, recipe_id):
"""Schedules a recipe for reprocessing and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param recipe_id: The id of the recipe
:type recipe_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
forced_nodes_json = rest_util.parse_dict(request, 'forced_nodes', required=True)
revision_num = rest_util.parse_dict(request, 'revision_num', required=False)
try:
forced_nodes = ForcedNodesV6(forced_nodes_json, do_validate=True)
except InvalidDiff as ex:
logger.exception('Unable to reprocess recipe. Invalid input: %s', forced_nodes_json)
raise BadParameter(unicode(ex))
try:
recipe = Recipe.objects.select_related('recipe_type').get(id=recipe_id)
if revision_num:
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, revision_num)
else:
revision_num = recipe.recipe_type.revision_num
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, recipe.recipe_type.revision_num)
except Recipe.DoesNotExist:
raise Http404
except RecipeTypeRevision.DoesNotExist:
raise Http404
if recipe.is_superseded:
raise BadParameter('Cannot reprocess a superseded recipe')
validation = recipe.recipe_type_rev.validate_forced_nodes(forced_nodes_json)
if not validation.is_valid:
raise BadParameter('Unable to reprocess recipe. Errors in validating forced_nodes: %s' % validation.errors)
if validation.warnings:
logger.warning('Warnings encountered when reprocessing: %s' % validation.warnings)
event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now())
root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id
recipe_type_name = recipe.recipe_type.name
# Execute all of the messages to perform the reprocess
messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id,
forced_nodes=forced_nodes.get_forced_nodes())
CommandMessageManager().send_messages(messages)
return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request):
"""Creates a new job, places it on the queue, and returns the new job information in JSON form
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
job_type_id = rest_util.parse_int(request, 'job_type_id')
job_data = rest_util.parse_dict(request, 'input', {})
configuration_dict = rest_util.parse_dict(request, 'configuration', required=False)
configuration = None
try:
jobData = DataV6(job_data, do_validate=True)
except InvalidData as ex:
logger.exception('Unable to queue new job. Invalid input: %s', job_data)
raise BadParameter(unicode(ex))
try:
job_type = JobType.objects.get(pk=job_type_id)
except JobType.DoesNotExist:
raise Http404
if configuration_dict:
try:
existing = convert_config_to_v6_json(job_type.get_job_configuration())
configuration = JobConfigurationV6(configuration_dict, existing=existing,
do_validate=True).get_configuration()
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
try:
job_id = Queue.objects.queue_new_job_for_user_v6(job_type=job_type, job_data=jobData.get_data(),
job_configuration=configuration)
CommandMessageManager().send_messages(create_process_job_input_messages([job_id]))
except InvalidData as err:
logger.exception('Invalid job data.')
return Response('Invalid job data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST)
try:
job_details = Job.objects.get_details(job_id)
except Job.DoesNotExist:
raise Http404
serializer = JobDetailsSerializerV6(job_details)
job_url = reverse('job_details_view', args=[job_id], request=request)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_url))
def process_ingested_source_file_cm(self, ingest_id, source, source_file,
when):
"""Processes the given ingested source file by kicking off its recipe.
All database changes are made in an atomic transaction.
:param source: The strike that triggered the ingest
:type scan: `object`
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
# Create the recipe handler associated with the ingest strike/scan
source_recipe_config = source.configuration['recipe']
recipe_name = source_recipe_config['name']
recipe_revision = source_recipe_config[
'revision_num'] if 'revision_num' in source_recipe_config else None
recipe_type = RecipeType.objects.get(name=recipe_name)
if recipe_revision:
recipe_type = RecipeTypeRevision.objects.get_revision(
recipe_name, recipe_revision).recipe_type
if len(recipe_type.get_definition().get_input_keys()) == 0:
logger.info(
'No inputs defined for recipe %s. Recipe will not be run.' %
recipe_name)
return
if recipe_type and recipe_type.is_active:
# Assuming one input per recipe, so pull the first defined input you find
recipe_data = Data()
input_name = recipe_type.get_definition().get_input_keys()[0]
recipe_data.add_value(FileValue(input_name, [source_file.id]))
event = self._create_trigger_event(source, source_file, when)
ingest_event = self._create_ingest_event(ingest_id, source,
source_file, when)
# This can cause a race condition with a slow DB.
messages = create_recipes_messages(
recipe_type.name, recipe_type.revision_num,
convert_data_to_v6_json(recipe_data).get_dict(), event.id,
ingest_event.id)
CommandMessageManager().send_messages(messages)
else:
logger.info(
'No recipe type found for %s %s or recipe type is inactive' %
(recipe_name, recipe_revision))
def test_failing_process_message(self, send_downstream, extract_command):
"""Validate logic for a process message failing in process execution"""
message = {'type': 'test', 'body': 'payload'}
manager = CommandMessageManager()
command = MagicMock()
command.execute = MagicMock(return_value=False)
extract_command.return_value = command
with self.assertRaises(CommandMessageExecuteFailure):
manager._process_message(message)
self.assertFalse(send_downstream.called)
def test_send_message(self):
"""Validate that send_message composes dict with `type` and `body` keys for backend"""
command = MagicMock(type='test')
command.to_json.return_value = 'body_content'
manager = CommandMessageManager()
send_messages = MagicMock()
backend = MagicMock(send_messages=send_messages)
manager._backend = backend
manager.send_messages([command])
send_messages.assert_called_with([{
'type': 'test',
'body': 'body_content'
}])
def test_successful_process_message(self, send_downstream,
extract_command):
"""Validate logic for a successful command process """
message = {'type': 'test', 'body': 'payload'}
manager = CommandMessageManager()
command = MagicMock(execute=MagicMock(return_value=True))
command.execute.return_value = True
command.new_messages = []
extract_command.return_value = command
manager._process_message(message)
send_downstream.assert_called_with([])
def patch_impl_v6(self, request, strike_id):
"""Edits an existing Strike process and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param strike_id: The ID of the Strike process
:type strike_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
configuration = rest_util.parse_dict(request, 'configuration', required=False)
config = None
try:
if configuration:
config = StrikeConfigurationV6(configuration, do_validate=True).get_configuration()
except InvalidStrikeConfiguration as ex:
raise BadParameter('Strike configuration invalid: %s' % unicode(ex))
try:
# must collect old config before editing strike
if config:
new_config = config.get_dict()
old_config = Strike.objects.get_details(strike_id)
Strike.objects.edit_strike(strike_id, title, description, config)
# if workspace has changed strike job must be restarted for changes to take effect
if config and old_config.configuration["workspace"] != new_config["workspace"]:
strike_job = old_config.job
Job.objects.update_jobs_to_canceled([strike_job], timezone.now())
requeue_jobs = []
requeue_jobs.append(QueuedJob(strike_job.id, strike_job.num_exes))
msg = create_requeue_jobs_messages(requeue_jobs)
CommandMessageManager().send_messages(msg)
except Strike.DoesNotExist:
raise Http404
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to edit Strike process: %s', strike_id)
raise BadParameter(unicode(ex))
return Response(status=status.HTTP_204_NO_CONTENT)
def post(self, request):
"""Kicks off the process of purging a given source file from Scale
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
if self.request.version != 'v6' and self.request.version != 'v7':
content = 'This endpoint is supported with REST API v6+'
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
file_id = rest_util.parse_int(request, 'file_id')
try:
file_id = int(file_id)
except ValueError:
content = 'The given file_id is not valid: %i' % (file_id)
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
# Attempt to fetch the ScaleFile model
try:
source_file = ScaleFile.objects.get(id=file_id)
except ScaleFile.DoesNotExist:
content = 'No file record exists for the given file_id: %i' % (
file_id)
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
# Inspect the file to ensure it will purge correctly
if source_file.file_type != 'SOURCE':
content = 'The given file_id does not correspond to a SOURCE file_type: %i' % (
file_id)
return Response(status=status.HTTP_400_BAD_REQUEST, data=content)
event = TriggerEvent.objects.create_trigger_event(
'USER', None, {'user': '******'}, now())
PurgeResults.objects.create(source_file_id=file_id,
trigger_event=event)
CommandMessageManager().send_messages([
create_purge_source_file_message(source_file_id=file_id,
trigger_id=event.id)
])
return Response(status=status.HTTP_204_NO_CONTENT)
def test_receive_message(self):
"""Validate the receive_message calls _process_message with each result"""
mocks = [MagicMock() for _ in range(10)]
def gen():
for mock in mocks:
yield mock
manager = CommandMessageManager()
manager._backend = MagicMock()
process_message = manager._process_message = MagicMock()
manager._backend.receive_messages = MagicMock(return_value=gen())
manager.receive_messages()
calls = [call(x) for x in mocks]
process_message.assert_has_calls(calls)
self.assertEquals(process_message.call_count, 10)
def post(self, request):
"""Submit command message to re-queue jobs that fit the given filter criteria
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
error_categories = rest_util.parse_string_list(request, 'error_categories', required=False)
error_ids = rest_util.parse_int_list(request, 'error_ids', required=False)
job_ids = rest_util.parse_int_list(request, 'job_ids', required=False)
job_status = rest_util.parse_string(request, 'status', required=False)
job_type_ids = rest_util.parse_int_list(request, 'job_type_ids', required=False)
priority = rest_util.parse_int(request, 'priority', required=False)
job_type_names = rest_util.parse_string_list(request, 'job_type_names', required=False)
batch_ids = rest_util.parse_int_list(request, 'batch_ids', required=False)
recipe_ids = rest_util.parse_int_list(request, 'recipe_ids', required=False)
is_superseded = rest_util.parse_bool(request, 'is_superseded', required=False)
job_types = rest_util.parse_dict_list(request, 'job_types', required=False)
for jt in job_types:
if 'name' not in jt or 'version' not in jt:
raise BadParameter('Job types argument invalid: %s' % job_types)
existing_job_type = JobType.objects.filter(name=jt['name'], version=jt['version']).first()
if not existing_job_type:
raise BadParameter(
'Job Type with name: %s and version: %s does not exist' % (jt['name'], jt['version']))
job_type_ids.append(existing_job_type.id)
# Create and send message
msg = create_requeue_jobs_bulk_message(started=started, ended=ended, error_categories=error_categories,
error_ids=error_ids, job_ids=job_ids, job_type_ids=job_type_ids,
priority=priority, status=job_status, job_type_names=job_type_names,
batch_ids=batch_ids, recipe_ids=recipe_ids, is_superseded=is_superseded)
CommandMessageManager().send_messages([msg])
return Response(status=status.HTTP_202_ACCEPTED)
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the command.
"""
logger.info('Command starting: scale_message_handler')
self.running = True
# Set the signal handler
signal.signal(signal.SIGINT, self.interupt)
signal.signal(signal.SIGTERM, self.interupt)
manager = CommandMessageManager()
while self.running:
manager.receive_messages()
logger.info('Command completed: scale_message_handler')
def initialize_system():
"""Performs any necessary functions needed for initializing Scale"""
logger.info('Initializing system')
Scheduler.objects.initialize_scheduler()
# Make sure clock job has been created
clock_job_type = JobType.objects.get_clock_job_type()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
if not count:
logger.info('Queuing Scale Clock job')
with transaction.atomic():
init_event = TriggerEvent.objects.create_trigger_event(
'SCALE_INIT', None, {}, now())
job = Queue.objects.queue_new_job_v6(clock_job_type, Data(),
init_event)
CommandMessageManager().send_messages(
create_process_job_input_messages([job.id]))
def handle_job_cancellation(self, job_id, when):
"""Handles the cancellation of a job. All database changes occur in an atomic transaction.
:param job_id: The ID of the job to be canceled
:type job_id: int
:param when: When the job was canceled
:type when: :class:`datetime.datetime`
"""
from recipe.messages.update_recipe import create_update_recipe_messages_from_node
Job.objects.update_jobs_to_canceled_old([job_id], when)
self.cancel_queued_jobs([job_id])
# Create and send messages to update dependent jobs so that they are BLOCKED
job = Job.objects.get(id=job_id)
if job.root_recipe_id:
msgs = create_update_recipe_messages_from_node(
[job.root_recipe_id])
CommandMessageManager().send_messages(msgs)
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the Scale batch creation process.
"""
batch_id = options.get('batch_id')
logger.info('Command starting: scale_batch_creator - Batch ID: %i',
batch_id)
# Schedule all the batch recipes
try:
batch = Batch.objects.get(id=batch_id)
CommandMessageManager().send_messages(
[create_batch_recipes_message(batch_id)])
except Batch.DoesNotExist:
logger.exception('Unable to find batch: %i', batch_id)
sys.exit(1)
logger.info('Command completed: scale_batch_creator')
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the command.
"""
body = options.get('body')
type = options.get('type')
count = options.get('count')
if not count:
count = 1
logger.info('Command starting: scale_send_message')
Message = get_message_type(type)
manager = CommandMessageManager()
messages = [Message.from_json(body) for _ in range(count)]
manager.send_messages(messages)
logger.info('Command completed: scale_send_message')
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the file destruction process.
"""
# Register a listener to handle clean shutdowns
signal.signal(signal.SIGTERM, self._onsigterm)
files_list = json.loads(os.environ.get('FILES'))
workspaces_list = json.loads(os.environ.get('WORKSPACES'))
job_id = int(os.environ.get('JOB_ID'))
trigger_id = int(os.environ.get('TRIGGER_ID'))
source_file_id = int(os.environ.get('SOURCE_FILE_ID'))
purge = os.environ.get('PURGE',
'true').lower() in ('yes', 'true', 't', '1')
workspaces = self._configure_workspaces(workspaces_list)
files = self._configure_files(files_list, workspaces)
logger.info('Command starting: scale_delete_files')
logger.info('File IDs: %s', [x.id for x in files])
for wrkspc_name, wrkspc in workspaces.iteritems():
delete_files_job.delete_files(
files=[f for f in files if f.workspace == wrkspc_name],
broker=wrkspc['broker'],
volume_path=wrkspc['volume_path'])
messages = create_delete_files_messages(files=files,
job_id=job_id,
trigger_id=trigger_id,
source_file_id=source_file_id,
purge=purge)
CommandMessageManager().send_messages(messages)
logger.info('Command completed: scale_delete_files')
sys.exit(0)
def queue_new_recipe_v6(self,
recipe_type,
recipe_input,
event,
ingest_event=None,
recipe_config=None,
batch_id=None,
superseded_recipe=None):
"""Creates a new recipe for the given type and data. and queues any of its jobs that are ready to run. If the
new recipe is superseding an old recipe, superseded_recipe, delta, and superseded_jobs must be provided and the
caller must have obtained a model lock on all job models in superseded_jobs and on the superseded_recipe model.
All database changes occur in an atomic transaction.
:param recipe_type: The type of the new recipe to create
:type recipe_type: :class:`recipe.models.RecipeType`
:param recipe_input: The recipe data to run on, should be None if superseded_recipe is provided
:type recipe_input: :class:`data.data.data.data`
:param event: The event that triggered the creation of this recipe
:type event: :class:`trigger.models.TriggerEvent`
:param recipe_config: config of the recipe, possibly None
:type recipe_config: :class:`recipe.configuration.configuration.RecipeConfiguration`
:param batch_id: The ID of the batch that contains this recipe
:type batch_id: int
:param superseded_recipe: The recipe that the created recipe is superseding, possibly None
:type superseded_recipe: :class:`recipe.models.Recipe`
:returns: New recipe type
:rtype: :class:`recipe.models.Recipe`
:raises :class:`recipe.configuration.data.exceptions.InvalidRecipeData`: If the recipe data is invalid
:raises :class:`recipe.exceptions.InactiveRecipeType`: If the recipe type is inactive
"""
recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.name, recipe_type.revision_num)
ingest_event_pk = None
event_pk = None
if ingest_event:
ingest_event_pk = ingest_event.pk
if event:
event_pk = event.pk
recipe = None
with transaction.atomic():
recipe = Recipe.objects.create_recipe_v6(
recipe_type_rev=recipe_type_rev,
event_id=event_pk,
ingest_id=ingest_event_pk,
input_data=recipe_input,
recipe_config=recipe_config,
batch_id=batch_id,
superseded_recipe=superseded_recipe)
recipe.save()
# If root_recipe_id is allowed to be None moving forword then a recipe will never complete.
# The recipe pk is only accessible once the recipe is saved (L#442).
if not recipe.recipe_id and not recipe.root_recipe_id:
recipe.root_recipe_id = recipe.pk
recipe.save()
# This can cause a race condition with a slow DB.
if recipe:
CommandMessageManager().send_messages(
create_process_recipe_input_messages([recipe.id]))
return recipe