def _generate_msg_queue_status(self): """Generates the Message Queue status message AMQP (rabbitmq) """ status_dict = {'OK': False, 'detail': {}, 'errors': [], 'warnings': []} status_dict['detail']['msg'] = 'Message Queue is healthy' status_dict['detail']['queue_name'] = scale_settings.QUEUE_NAME status_dict['detail']['num_message_handlers'] = scheduler_mgr.config.num_message_handlers status_dict['detail']['queue_depth'] = 0 status_dict['detail']['region_name'] = '' status_dict['detail']['type'] = '' try: broker_details = BrokerDetails.from_broker_url(scale_settings.BROKER_URL) except InvalidBrokerUrl: msg = 'Error parsing broker url' status_dict['errors'] = [{'INVALID_BROKER_URL': msg}] status_dict['detail']['msg'] = msg return status_dict status_dict['detail']['type'] = broker_details.get_type() if broker_details.get_type() == 'amqp': try: CommandMessageManager().get_queue_size() status_dict['OK'] = True except Exception as ex: logger.error('Error connecting to RabbitMQ: %s' % unicode(ex)) status_dict['OK'] = False msg = 'Error connecting to RabbitMQ: Check Logs for details' status_dict['errors'] = [{'RABBITMQ_ERROR': msg}] status_dict['detail']['msg'] = 'Unable to get message queue size.' elif broker_details.get_type() == 'sqs': status_dict['detail']['region_name'] = broker_details.get_address() try: CommandMessageManager().get_queue_size() status_dict['OK'] = True except Exception as ex: logger.error('Unable to get queue size from sqs: %s' % unicode(ex)) msg = 'Error connecting to SQS: Check Logs for details' status_dict['OK'] = False status_dict['errors'] = [{'SQS_ERROR': msg}] status_dict['detail']['msg'] = 'Unable to get message queue size.' else: status_dict['OK'] = False status_dict['detail']['msg'] = 'Broker is an unsupported type: %s' % broker_details.get_type() if status_dict['OK']: status_dict['detail']['queue_depth'] = CommandMessageManager().get_queue_size() if scale_settings.MESSSAGE_QUEUE_DEPTH_WARN > 0 and status_dict['detail']['queue_depth'] > scale_settings.MESSSAGE_QUEUE_DEPTH_WARN: status_dict['warnings'].append({'LARGE_QUEUE': 'Message queue is very large'}) status_dict['detail']['msg'] = 'Message queue is large. Scale may be unresponsive.' return status_dict
def process_event(self, event, last_event=None): """See :meth:`job.clock.ClockEventProcessor.process_event`. Compares the new event with the last event any missing metrics jobs. """ # Attempt to get the daily metrics job type try: job_type = JobType.objects.filter( name='scale-daily-metrics').last() except JobType.DoesNotExist: raise ClockEventError( 'Missing required job type: scale-daily-metrics') if last_event: # Build a list of days that require metrics day_count = xrange( (event.occurred.date() - last_event.occurred.date()).days) days = [ last_event.occurred.date() + datetime.timedelta(days=d) for d in day_count ] else: # Use the previous day when first triggered days = [timezone.now().date() - datetime.timedelta(days=1)] # Schedule one job for each required day for day in days: job_data = Data() job_data.add_value(JsonValue('DAY', day.strftime('%Y-%m-%d'))) job = Queue.objects.queue_new_job_v6(job_type, job_data, event) CommandMessageManager().send_messages( create_process_job_input_messages([job.id]))
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. """ count = options.get('count') if not count: count = 1 logger.info( 'Command starting: scale_echo_message - sending {} message(s)'. format(count)) manager = CommandMessageManager() messages = [] for x in range(count): messages.append( EchoCommandMessage.from_json({ 'message': 'Greetings, this is echo #{} at {}!'.format( x + 1, datetime.utcnow()) })) manager.send_messages(messages) logger.info('Command completed: scale_echo_message')
def process_manual_ingested_source_file(self, ingest_id, source_file, when, recipe_type_id): """Processes a manual ingest where a strike or scan is not involved. All database changes are made in an atomic transaction :param ingest_id: :type ingest_id: int :param source_file: The source file that was ingested :type source_file: :class:`source.models.SourceFile` :param when: When the source file was ingested :type when: :class:`datetime.datetime` :param recipe_type_id: id of the Recipe type to kick off :type recipe_type_id: int """ recipe_type = RecipeType.objects.get(id=recipe_type_id) if recipe_type and recipe_type.is_active: recipe_data = Data() input_name = recipe_type.get_definition().get_input_keys()[0] recipe_data.add_value(FileValue(input_name, [source_file.id])) event = self._create_trigger_event(None, source_file, when) ingest_event = self._create_ingest_event(ingest_id, None, source_file, when) messages = create_recipes_messages( recipe_type.name, recipe_type.revision_num, convert_data_to_v6_json(recipe_data).get_dict(), event.id, ingest_event.id) CommandMessageManager().send_messages(messages) else: logger.info( 'No recipe type found for id %s or recipe type is inactive' % recipe_type_id)
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the file destruction process. """ # Register a listener to handle clean shutdowns signal.signal(signal.SIGTERM, self._onsigterm) files = options.get('files') workspace_list = options.get('workspace') purge = options.get('purge') workspaces = self._detect_workspaces(workspace_list) files = self._configure_files(files, workspaces) logger.info('Command starting: scale_delete_files') logger.info('File IDs: %s', [x.id for x in files]) for wrkspc_name, wrkspc in workspaces.iteritems(): delete_files_job.delete_files( files=[f for f in files if f.workspace == wrkspc_name], broker=wrkspc['broker'], volume_path=wrkspc['volume_path']) messages = create_delete_files_messages(files=files, purge=purge) CommandMessageManager().send_messages(messages) logger.info('Command completed: scale_delete_files') sys.exit(0)
def test_no_message_send_downstream(self, send_messages): """Validate send_message is not called when messages is empty""" manager = CommandMessageManager() manager._send_downstream([]) self.assertFalse(send_messages.called)
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. """ logger.info('Command starting: scale_message_handler') self.running = True logger.info('Initializing message handler') logger.info('Caching builtin errors...') Error.objects.cache_builtin_errors() logger.info('Initialization complete, ready to process messages') # Set the signal handler signal.signal(signal.SIGINT, self.interupt) signal.signal(signal.SIGTERM, self.interupt) manager = CommandMessageManager() while self.running: manager.receive_messages() logger.info('Command completed: scale_message_handler')
def __init__(self): """Constructor """ super(MessagingThread, self).__init__('Messaging', THROTTLE, WARN_THRESHOLD) self._manager = CommandMessageManager() self._messages = []
def _create_v6(self, request): """The v6 version for creating batches :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') definition_dict = rest_util.parse_dict(request, 'definition') configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %d' % recipe_type_id) # Validate and create the batch try: definition = BatchDefinitionV6(definition=definition_dict, do_validate=True).get_definition() configuration = BatchConfigurationV6( configuration=configuration_dict, do_validate=True).get_configuration() with transaction.atomic(): event = TriggerEvent.objects.create_trigger_event( 'USER', None, {'user': '******'}, now()) batch = Batch.objects.create_batch_v6( title, description, recipe_type, event, definition, configuration=configuration) CommandMessageManager().send_messages( [create_batch_recipes_message(batch.id)]) except InvalidDefinition as ex: raise BadParameter(unicode(ex)) except InvalidConfiguration as ex: raise BadParameter(unicode(ex)) # Fetch the full batch with details batch = Batch.objects.get_details_v6(batch.id) url = reverse('batch_details_view', args=[batch.id], request=request) serializer = BatchDetailsSerializerV6(batch) return Response(serializer.data, status=status.HTTP_201_CREATED, headers={'Location': url})
def test_successful_send_downstream(self, send_messages): """Validate call of send_message for each downstream message""" messages = ['one', 'two'] manager = CommandMessageManager() manager._send_downstream(messages) send_messages.assert_called_with(messages)
def test_send_message_no_serializer(self): """Validate that send_message raises AttributeError when to_json is not available""" class InvalidCommand(object): def __init__(self): self.type = 'test' message = InvalidCommand() manager = CommandMessageManager() with self.assertRaises(AttributeError): manager.send_messages([message])
def test_send_messages_no_type(self): """Validate that send_message raises AttributeError when message type is not available""" class InvalidCommand(object): def to_json(self): # pragma: no cover pass message = InvalidCommand() manager = CommandMessageManager() with self.assertRaises(AttributeError): manager.send_messages([message])
def test_receive_message_execute_failure(self, process, broker_details, get_message_backend): """Exercise all exception code paths within receive_message""" def gen(): yield MagicMock() process.side_effect = CommandMessageExecuteFailure manager = CommandMessageManager() manager._backend = MagicMock() manager._backend.receive_messages.return_value = gen() manager.receive_messages()
def _post_v6(self, request, recipe_id): """Schedules a recipe for reprocessing and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The id of the recipe :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ forced_nodes_json = rest_util.parse_dict(request, 'forced_nodes', required=True) revision_num = rest_util.parse_dict(request, 'revision_num', required=False) try: forced_nodes = ForcedNodesV6(forced_nodes_json, do_validate=True) except InvalidDiff as ex: logger.exception('Unable to reprocess recipe. Invalid input: %s', forced_nodes_json) raise BadParameter(unicode(ex)) try: recipe = Recipe.objects.select_related('recipe_type').get(id=recipe_id) if revision_num: recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, revision_num) else: revision_num = recipe.recipe_type.revision_num recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, recipe.recipe_type.revision_num) except Recipe.DoesNotExist: raise Http404 except RecipeTypeRevision.DoesNotExist: raise Http404 if recipe.is_superseded: raise BadParameter('Cannot reprocess a superseded recipe') validation = recipe.recipe_type_rev.validate_forced_nodes(forced_nodes_json) if not validation.is_valid: raise BadParameter('Unable to reprocess recipe. Errors in validating forced_nodes: %s' % validation.errors) if validation.warnings: logger.warning('Warnings encountered when reprocessing: %s' % validation.warnings) event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now()) root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id recipe_type_name = recipe.recipe_type.name # Execute all of the messages to perform the reprocess messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id, forced_nodes=forced_nodes.get_forced_nodes()) CommandMessageManager().send_messages(messages) return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request): """Creates a new job, places it on the queue, and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_type_id = rest_util.parse_int(request, 'job_type_id') job_data = rest_util.parse_dict(request, 'input', {}) configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: jobData = DataV6(job_data, do_validate=True) except InvalidData as ex: logger.exception('Unable to queue new job. Invalid input: %s', job_data) raise BadParameter(unicode(ex)) try: job_type = JobType.objects.get(pk=job_type_id) except JobType.DoesNotExist: raise Http404 if configuration_dict: try: existing = convert_config_to_v6_json(job_type.get_job_configuration()) configuration = JobConfigurationV6(configuration_dict, existing=existing, do_validate=True).get_configuration() except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) try: job_id = Queue.objects.queue_new_job_for_user_v6(job_type=job_type, job_data=jobData.get_data(), job_configuration=configuration) CommandMessageManager().send_messages(create_process_job_input_messages([job_id])) except InvalidData as err: logger.exception('Invalid job data.') return Response('Invalid job data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) try: job_details = Job.objects.get_details(job_id) except Job.DoesNotExist: raise Http404 serializer = JobDetailsSerializerV6(job_details) job_url = reverse('job_details_view', args=[job_id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_url))
def process_ingested_source_file_cm(self, ingest_id, source, source_file, when): """Processes the given ingested source file by kicking off its recipe. All database changes are made in an atomic transaction. :param source: The strike that triggered the ingest :type scan: `object` :param source_file: The source file that was ingested :type source_file: :class:`source.models.SourceFile` :param when: When the source file was ingested :type when: :class:`datetime.datetime` """ # Create the recipe handler associated with the ingest strike/scan source_recipe_config = source.configuration['recipe'] recipe_name = source_recipe_config['name'] recipe_revision = source_recipe_config[ 'revision_num'] if 'revision_num' in source_recipe_config else None recipe_type = RecipeType.objects.get(name=recipe_name) if recipe_revision: recipe_type = RecipeTypeRevision.objects.get_revision( recipe_name, recipe_revision).recipe_type if len(recipe_type.get_definition().get_input_keys()) == 0: logger.info( 'No inputs defined for recipe %s. Recipe will not be run.' % recipe_name) return if recipe_type and recipe_type.is_active: # Assuming one input per recipe, so pull the first defined input you find recipe_data = Data() input_name = recipe_type.get_definition().get_input_keys()[0] recipe_data.add_value(FileValue(input_name, [source_file.id])) event = self._create_trigger_event(source, source_file, when) ingest_event = self._create_ingest_event(ingest_id, source, source_file, when) # This can cause a race condition with a slow DB. messages = create_recipes_messages( recipe_type.name, recipe_type.revision_num, convert_data_to_v6_json(recipe_data).get_dict(), event.id, ingest_event.id) CommandMessageManager().send_messages(messages) else: logger.info( 'No recipe type found for %s %s or recipe type is inactive' % (recipe_name, recipe_revision))
def test_failing_process_message(self, send_downstream, extract_command): """Validate logic for a process message failing in process execution""" message = {'type': 'test', 'body': 'payload'} manager = CommandMessageManager() command = MagicMock() command.execute = MagicMock(return_value=False) extract_command.return_value = command with self.assertRaises(CommandMessageExecuteFailure): manager._process_message(message) self.assertFalse(send_downstream.called)
def test_send_message(self): """Validate that send_message composes dict with `type` and `body` keys for backend""" command = MagicMock(type='test') command.to_json.return_value = 'body_content' manager = CommandMessageManager() send_messages = MagicMock() backend = MagicMock(send_messages=send_messages) manager._backend = backend manager.send_messages([command]) send_messages.assert_called_with([{ 'type': 'test', 'body': 'body_content' }])
def test_successful_process_message(self, send_downstream, extract_command): """Validate logic for a successful command process """ message = {'type': 'test', 'body': 'payload'} manager = CommandMessageManager() command = MagicMock(execute=MagicMock(return_value=True)) command.execute.return_value = True command.new_messages = [] extract_command.return_value = command manager._process_message(message) send_downstream.assert_called_with([])
def patch_impl_v6(self, request, strike_id): """Edits an existing Strike process and returns the updated details :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param strike_id: The ID of the Strike process :type strike_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) configuration = rest_util.parse_dict(request, 'configuration', required=False) config = None try: if configuration: config = StrikeConfigurationV6(configuration, do_validate=True).get_configuration() except InvalidStrikeConfiguration as ex: raise BadParameter('Strike configuration invalid: %s' % unicode(ex)) try: # must collect old config before editing strike if config: new_config = config.get_dict() old_config = Strike.objects.get_details(strike_id) Strike.objects.edit_strike(strike_id, title, description, config) # if workspace has changed strike job must be restarted for changes to take effect if config and old_config.configuration["workspace"] != new_config["workspace"]: strike_job = old_config.job Job.objects.update_jobs_to_canceled([strike_job], timezone.now()) requeue_jobs = [] requeue_jobs.append(QueuedJob(strike_job.id, strike_job.num_exes)) msg = create_requeue_jobs_messages(requeue_jobs) CommandMessageManager().send_messages(msg) except Strike.DoesNotExist: raise Http404 except InvalidStrikeConfiguration as ex: logger.exception('Unable to edit Strike process: %s', strike_id) raise BadParameter(unicode(ex)) return Response(status=status.HTTP_204_NO_CONTENT)
def post(self, request): """Kicks off the process of purging a given source file from Scale :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ if self.request.version != 'v6' and self.request.version != 'v7': content = 'This endpoint is supported with REST API v6+' return Response(status=status.HTTP_400_BAD_REQUEST, data=content) file_id = rest_util.parse_int(request, 'file_id') try: file_id = int(file_id) except ValueError: content = 'The given file_id is not valid: %i' % (file_id) return Response(status=status.HTTP_400_BAD_REQUEST, data=content) # Attempt to fetch the ScaleFile model try: source_file = ScaleFile.objects.get(id=file_id) except ScaleFile.DoesNotExist: content = 'No file record exists for the given file_id: %i' % ( file_id) return Response(status=status.HTTP_400_BAD_REQUEST, data=content) # Inspect the file to ensure it will purge correctly if source_file.file_type != 'SOURCE': content = 'The given file_id does not correspond to a SOURCE file_type: %i' % ( file_id) return Response(status=status.HTTP_400_BAD_REQUEST, data=content) event = TriggerEvent.objects.create_trigger_event( 'USER', None, {'user': '******'}, now()) PurgeResults.objects.create(source_file_id=file_id, trigger_event=event) CommandMessageManager().send_messages([ create_purge_source_file_message(source_file_id=file_id, trigger_id=event.id) ]) return Response(status=status.HTTP_204_NO_CONTENT)
def test_receive_message(self): """Validate the receive_message calls _process_message with each result""" mocks = [MagicMock() for _ in range(10)] def gen(): for mock in mocks: yield mock manager = CommandMessageManager() manager._backend = MagicMock() process_message = manager._process_message = MagicMock() manager._backend.receive_messages = MagicMock(return_value=gen()) manager.receive_messages() calls = [call(x) for x in mocks] process_message.assert_has_calls(calls) self.assertEquals(process_message.call_count, 10)
def post(self, request): """Submit command message to re-queue jobs that fit the given filter criteria :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) error_categories = rest_util.parse_string_list(request, 'error_categories', required=False) error_ids = rest_util.parse_int_list(request, 'error_ids', required=False) job_ids = rest_util.parse_int_list(request, 'job_ids', required=False) job_status = rest_util.parse_string(request, 'status', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_ids', required=False) priority = rest_util.parse_int(request, 'priority', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_names', required=False) batch_ids = rest_util.parse_int_list(request, 'batch_ids', required=False) recipe_ids = rest_util.parse_int_list(request, 'recipe_ids', required=False) is_superseded = rest_util.parse_bool(request, 'is_superseded', required=False) job_types = rest_util.parse_dict_list(request, 'job_types', required=False) for jt in job_types: if 'name' not in jt or 'version' not in jt: raise BadParameter('Job types argument invalid: %s' % job_types) existing_job_type = JobType.objects.filter(name=jt['name'], version=jt['version']).first() if not existing_job_type: raise BadParameter( 'Job Type with name: %s and version: %s does not exist' % (jt['name'], jt['version'])) job_type_ids.append(existing_job_type.id) # Create and send message msg = create_requeue_jobs_bulk_message(started=started, ended=ended, error_categories=error_categories, error_ids=error_ids, job_ids=job_ids, job_type_ids=job_type_ids, priority=priority, status=job_status, job_type_names=job_type_names, batch_ids=batch_ids, recipe_ids=recipe_ids, is_superseded=is_superseded) CommandMessageManager().send_messages([msg]) return Response(status=status.HTTP_202_ACCEPTED)
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. """ logger.info('Command starting: scale_message_handler') self.running = True # Set the signal handler signal.signal(signal.SIGINT, self.interupt) signal.signal(signal.SIGTERM, self.interupt) manager = CommandMessageManager() while self.running: manager.receive_messages() logger.info('Command completed: scale_message_handler')
def initialize_system(): """Performs any necessary functions needed for initializing Scale""" logger.info('Initializing system') Scheduler.objects.initialize_scheduler() # Make sure clock job has been created clock_job_type = JobType.objects.get_clock_job_type() count = Job.objects.filter(job_type_id=clock_job_type.id).count() if not count: logger.info('Queuing Scale Clock job') with transaction.atomic(): init_event = TriggerEvent.objects.create_trigger_event( 'SCALE_INIT', None, {}, now()) job = Queue.objects.queue_new_job_v6(clock_job_type, Data(), init_event) CommandMessageManager().send_messages( create_process_job_input_messages([job.id]))
def handle_job_cancellation(self, job_id, when): """Handles the cancellation of a job. All database changes occur in an atomic transaction. :param job_id: The ID of the job to be canceled :type job_id: int :param when: When the job was canceled :type when: :class:`datetime.datetime` """ from recipe.messages.update_recipe import create_update_recipe_messages_from_node Job.objects.update_jobs_to_canceled_old([job_id], when) self.cancel_queued_jobs([job_id]) # Create and send messages to update dependent jobs so that they are BLOCKED job = Job.objects.get(id=job_id) if job.root_recipe_id: msgs = create_update_recipe_messages_from_node( [job.root_recipe_id]) CommandMessageManager().send_messages(msgs)
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the Scale batch creation process. """ batch_id = options.get('batch_id') logger.info('Command starting: scale_batch_creator - Batch ID: %i', batch_id) # Schedule all the batch recipes try: batch = Batch.objects.get(id=batch_id) CommandMessageManager().send_messages( [create_batch_recipes_message(batch_id)]) except Batch.DoesNotExist: logger.exception('Unable to find batch: %i', batch_id) sys.exit(1) logger.info('Command completed: scale_batch_creator')
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. """ body = options.get('body') type = options.get('type') count = options.get('count') if not count: count = 1 logger.info('Command starting: scale_send_message') Message = get_message_type(type) manager = CommandMessageManager() messages = [Message.from_json(body) for _ in range(count)] manager.send_messages(messages) logger.info('Command completed: scale_send_message')
def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the file destruction process. """ # Register a listener to handle clean shutdowns signal.signal(signal.SIGTERM, self._onsigterm) files_list = json.loads(os.environ.get('FILES')) workspaces_list = json.loads(os.environ.get('WORKSPACES')) job_id = int(os.environ.get('JOB_ID')) trigger_id = int(os.environ.get('TRIGGER_ID')) source_file_id = int(os.environ.get('SOURCE_FILE_ID')) purge = os.environ.get('PURGE', 'true').lower() in ('yes', 'true', 't', '1') workspaces = self._configure_workspaces(workspaces_list) files = self._configure_files(files_list, workspaces) logger.info('Command starting: scale_delete_files') logger.info('File IDs: %s', [x.id for x in files]) for wrkspc_name, wrkspc in workspaces.iteritems(): delete_files_job.delete_files( files=[f for f in files if f.workspace == wrkspc_name], broker=wrkspc['broker'], volume_path=wrkspc['volume_path']) messages = create_delete_files_messages(files=files, job_id=job_id, trigger_id=trigger_id, source_file_id=source_file_id, purge=purge) CommandMessageManager().send_messages(messages) logger.info('Command completed: scale_delete_files') sys.exit(0)
def queue_new_recipe_v6(self, recipe_type, recipe_input, event, ingest_event=None, recipe_config=None, batch_id=None, superseded_recipe=None): """Creates a new recipe for the given type and data. and queues any of its jobs that are ready to run. If the new recipe is superseding an old recipe, superseded_recipe, delta, and superseded_jobs must be provided and the caller must have obtained a model lock on all job models in superseded_jobs and on the superseded_recipe model. All database changes occur in an atomic transaction. :param recipe_type: The type of the new recipe to create :type recipe_type: :class:`recipe.models.RecipeType` :param recipe_input: The recipe data to run on, should be None if superseded_recipe is provided :type recipe_input: :class:`data.data.data.data` :param event: The event that triggered the creation of this recipe :type event: :class:`trigger.models.TriggerEvent` :param recipe_config: config of the recipe, possibly None :type recipe_config: :class:`recipe.configuration.configuration.RecipeConfiguration` :param batch_id: The ID of the batch that contains this recipe :type batch_id: int :param superseded_recipe: The recipe that the created recipe is superseding, possibly None :type superseded_recipe: :class:`recipe.models.Recipe` :returns: New recipe type :rtype: :class:`recipe.models.Recipe` :raises :class:`recipe.configuration.data.exceptions.InvalidRecipeData`: If the recipe data is invalid :raises :class:`recipe.exceptions.InactiveRecipeType`: If the recipe type is inactive """ recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type.name, recipe_type.revision_num) ingest_event_pk = None event_pk = None if ingest_event: ingest_event_pk = ingest_event.pk if event: event_pk = event.pk recipe = None with transaction.atomic(): recipe = Recipe.objects.create_recipe_v6( recipe_type_rev=recipe_type_rev, event_id=event_pk, ingest_id=ingest_event_pk, input_data=recipe_input, recipe_config=recipe_config, batch_id=batch_id, superseded_recipe=superseded_recipe) recipe.save() # If root_recipe_id is allowed to be None moving forword then a recipe will never complete. # The recipe pk is only accessible once the recipe is saved (L#442). if not recipe.recipe_id and not recipe.root_recipe_id: recipe.root_recipe_id = recipe.pk recipe.save() # This can cause a race condition with a slow DB. if recipe: CommandMessageManager().send_messages( create_process_recipe_input_messages([recipe.id])) return recipe