def process_event(self, event, last_event=None):
"""See :meth:`job.clock.ClockEventProcessor.process_event`.
Compares the new event with the last event any missing metrics jobs.
"""
# Attempt to get the daily metrics job type
try:
job_type = JobType.objects.filter(
name='scale-daily-metrics').last()
except JobType.DoesNotExist:
raise ClockEventError(
'Missing required job type: scale-daily-metrics')
if last_event:
# Build a list of days that require metrics
day_count = xrange(
(event.occurred.date() - last_event.occurred.date()).days)
days = [
last_event.occurred.date() + datetime.timedelta(days=d)
for d in day_count
]
else:
# Use the previous day when first triggered
days = [timezone.now().date() - datetime.timedelta(days=1)]
# Schedule one job for each required day
for day in days:
job_data = Data()
job_data.add_value(JsonValue('DAY', day.strftime('%Y-%m-%d')))
job = Queue.objects.queue_new_job_v6(job_type, job_data, event)
CommandMessageManager().send_messages(
create_process_job_input_messages([job.id]))
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
with transaction.atomic():
self._perform_locking()
jobs = self._find_existing_jobs()
if not jobs:
try:
jobs = self._create_jobs()
except InactiveJobType as ex:
logger.exception('Attempting to create a job with an inactive job type: %s. Message will not re-run.', ex)
return True
process_input_job_ids = []
for job in jobs:
# process_input indicates if job is in a recipe and ready to get its input from its dependencies
process_input = self.recipe_id and self._process_input.get(job.id, False)
if job.has_input() or process_input:
# This new job is all ready to have its input processed
process_input_job_ids.append(job.id)
self.new_messages.extend(create_process_job_input_messages(process_input_job_ids))
if self.recipe_id:
# If these jobs belong to a recipe, update its metrics now that the jobs are created
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages
self.new_messages.extend(create_update_recipe_metrics_messages([self.recipe_id]))
return True
def post(self, request):
"""Creates a new job, places it on the queue, and returns the new job information in JSON form
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
job_type_id = rest_util.parse_int(request, 'job_type_id')
job_data = rest_util.parse_dict(request, 'input', {})
configuration_dict = rest_util.parse_dict(request, 'configuration', required=False)
configuration = None
try:
jobData = DataV6(job_data, do_validate=True)
except InvalidData as ex:
logger.exception('Unable to queue new job. Invalid input: %s', job_data)
raise BadParameter(unicode(ex))
try:
job_type = JobType.objects.get(pk=job_type_id)
except JobType.DoesNotExist:
raise Http404
if configuration_dict:
try:
existing = convert_config_to_v6_json(job_type.get_job_configuration())
configuration = JobConfigurationV6(configuration_dict, existing=existing,
do_validate=True).get_configuration()
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
try:
job_id = Queue.objects.queue_new_job_for_user_v6(job_type=job_type, job_data=jobData.get_data(),
job_configuration=configuration)
CommandMessageManager().send_messages(create_process_job_input_messages([job_id]))
except InvalidData as err:
logger.exception('Invalid job data.')
return Response('Invalid job data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST)
try:
job_details = Job.objects.get_details(job_id)
except Job.DoesNotExist:
raise Http404
serializer = JobDetailsSerializerV6(job_details)
job_url = reverse('job_details_view', args=[job_id], request=request)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_url))
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
from ingest.models import Ingest
ingest_job_type = Ingest.objects.get_ingest_job_type()
# Grab the ingest object
ingest = Ingest.objects.get(pk=self.ingest_id)
when = ingest.transfer_ended if ingest.transfer_ended else now()
desc = {'file_name': ingest.file_name}
event = None
ingest_id = ingest.id
with transaction.atomic():
# Create the appropriate triggerevent
if self.create_ingest_type == STRIKE_JOB_TYPE:
desc['strike_id'] = self.strike_id
event = TriggerEvent.objects.create_trigger_event('STRIKE_TRANSFER', None, desc, when)
elif self.create_ingest_type == SCAN_JOB_TYPE:
ingest_id = Ingest.objects.get(scan_id=self.scan_id, file_name=ingest.file_name).id
desc['scan_id'] = self.scan_id
event = TriggerEvent.objects.create_trigger_event('SCAN_TRANSFER', None, desc, when)
data = Data()
data.add_value(JsonValue('ingest_id', ingest_id))
data.add_value(JsonValue('workspace', ingest.workspace.name))
if ingest.new_workspace:
data.add_value(JsonValue('new_workspace', ingest.new_workspace.name))
ingest_job = None
with transaction.atomic():
ingest_job = Queue.objects.queue_new_job_v6(ingest_job_type, data, event)
ingest.job = ingest_job
ingest.status = 'QUEUED'
ingest.save()
# Send message to start processing job input (done outside the transaction to hope the job exists)
# This can cause a race condition with a slow DB.
job = Job.objects.get_details(ingest_job.id)
self.new_messages.extend(create_process_job_input_messages([job.id]))
return True
def initialize_system():
"""Performs any necessary functions needed for initializing Scale"""
logger.info('Initializing system')
Scheduler.objects.initialize_scheduler()
# Make sure clock job has been created
clock_job_type = JobType.objects.get_clock_job_type()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
if not count:
logger.info('Queuing Scale Clock job')
with transaction.atomic():
init_event = TriggerEvent.objects.create_trigger_event(
'SCALE_INIT', None, {}, now())
job = Queue.objects.queue_new_job_v6(clock_job_type, Data(),
init_event)
CommandMessageManager().send_messages(
create_process_job_input_messages([job.id]))
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
job_type_rev = JobTypeRevision.objects.get_revision(
self.job_type_name, self.job_type_version, self.job_type_rev_num)
# Check to see if jobs were already created so that message is idempotent
jobs = self._find_existing_jobs(job_type_rev)
if not jobs:
jobs = self._create_jobs(job_type_rev)
# If the jobs already have their input data or process_input flag is set (recipe is ready to pass input), send
# messages to process job input
if jobs and (self.input_data or self.process_input):
job_ids = [job.id for job in jobs]
self.new_messages.extend(
create_process_job_input_messages(job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
recipe = Recipe.objects.get_recipe_instance_from_root(
self.root_recipe_id)
recipe_model = recipe.recipe_model
when = now()
jobs_to_update = recipe.get_jobs_to_update()
blocked_job_ids = jobs_to_update['BLOCKED']
pending_job_ids = jobs_to_update['PENDING']
nodes_to_create = recipe.get_nodes_to_create()
nodes_to_process_input = recipe.get_nodes_to_process_input()
if not recipe_model.is_completed and recipe.has_completed():
Recipe.objects.complete_recipes([recipe_model.id], when)
# Create new messages for changing job statuses
if len(blocked_job_ids):
logger.info('Found %d job(s) that should transition to BLOCKED',
len(blocked_job_ids))
self.new_messages.extend(
create_blocked_jobs_messages(blocked_job_ids, when))
if len(pending_job_ids):
logger.info('Found %d job(s) that should transition to PENDING',
len(pending_job_ids))
self.new_messages.extend(
create_pending_jobs_messages(pending_job_ids, when))
# Create new messages to create recipe nodes
conditions = []
recipe_jobs = []
subrecipes = []
for node_name, node_def in nodes_to_create.items():
process_input = False
if node_name in nodes_to_process_input:
process_input = True
del nodes_to_process_input[node_name]
if node_def.node_type == ConditionNodeDefinition.NODE_TYPE:
condition = Condition(node_name, process_input)
conditions.append(condition)
elif node_def.node_type == JobNodeDefinition.NODE_TYPE:
job = RecipeJob(node_def.job_type_name,
node_def.job_type_version,
node_def.revision_num, node_name,
process_input)
recipe_jobs.append(job)
elif node_def.node_type == RecipeNodeDefinition.NODE_TYPE:
subrecipe = SubRecipe(node_def.recipe_type_name,
node_def.revision_num, node_name,
process_input)
subrecipes.append(subrecipe)
if len(conditions):
logger.info('Found %d condition(s) to create for this recipe',
len(conditions))
self.new_messages.extend(
create_conditions_messages(recipe_model, conditions))
if len(recipe_jobs):
logger.info('Found %d job(s) to create for this recipe',
len(recipe_jobs))
self.new_messages.extend(
create_jobs_messages_for_recipe(recipe_model, recipe_jobs))
if len(subrecipes):
logger.info('Found %d sub-recipe(s) to create for this recipe',
len(subrecipes))
self.new_messages.extend(
create_subrecipes_messages(recipe_model,
subrecipes,
forced_nodes=self.forced_nodes))
# Create new messages for processing recipe node input
process_condition_ids = []
process_job_ids = []
process_recipe_ids = []
for node_name, node in nodes_to_process_input.items():
if node.node_type == ConditionNodeDefinition.NODE_TYPE:
process_condition_ids.append(node.condition.id)
elif node.node_type == JobNodeDefinition.NODE_TYPE:
process_job_ids.append(node.job.id)
elif node.node_type == RecipeNodeDefinition.NODE_TYPE:
process_recipe_ids.append(node.recipe.id)
if len(process_condition_ids):
logger.info('Found %d condition(s) to process their input',
len(process_condition_ids))
self.new_messages.extend(
create_process_condition_messages(process_condition_ids))
if len(process_job_ids):
logger.info(
'Found %d job(s) to process their input and move to the queue',
len(process_job_ids))
self.new_messages.extend(
create_process_job_input_messages(process_job_ids))
if len(process_recipe_ids):
logger.info(
'Found %d sub-recipe(s) to process their input and begin processing',
len(process_recipe_ids))
self.new_messages.extend(
create_process_recipe_input_messages(process_recipe_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
when = now()
blocked_job_ids = set()
pending_job_ids = set()
completed_recipe_ids = []
updated_batch_ids = []
num_jobs_with_input = 0
job_ids_ready_for_first_queue = []
with transaction.atomic():
# Lock recipes
Recipe.objects.get_locked_recipes(self._recipe_ids)
# Process all recipe handlers
recipes = list(Recipe.objects.get_recipes_with_definitions(self._recipe_ids))
while len(recipes) > 0:
# Gather up a list of recipes that doesn't exceed the job number limit
recipe_list = [recipes.pop()]
# TODO: eventually use job count per recipe, right now assume 10
num_jobs = 10
while len(recipes) > 0 and num_jobs + 10 < MAX_JOBS_AT_A_TIME:
num_jobs += 10
recipe_list.append(recipes.pop())
# Process handlers for the list of recipes
handlers = Recipe.objects.get_recipe_handlers(recipe_list)
# Create any jobs needed
jobs_to_create = []
recipe_handler_dicts = {}
for handler in handlers:
handler_dict = handler.get_jobs_to_create()
recipe_handler_dicts[handler.recipe.id] = handler_dict
for job_list in handler_dict.values():
for job in job_list:
jobs_to_create.append(job)
if jobs_to_create:
Job.objects.bulk_create(jobs_to_create)
recipe_jobs_to_create = []
for handler in handlers:
recipe_id = handler.recipe.id
handler_dict = recipe_handler_dicts[recipe_id]
recipe_jobs = []
for job_name, job_list in handler_dict.iteritems():
for job in job_list:
recipe_job = RecipeNode()
recipe_job.job = job
recipe_job.node_name = job_name
recipe_job.recipe_id = recipe_id
recipe_jobs.append(recipe_job)
recipe_jobs_to_create.extend(recipe_jobs)
handler.add_jobs(recipe_jobs)
if recipe_jobs_to_create:
RecipeNode.objects.bulk_create(recipe_jobs_to_create)
for handler in handlers:
for blocked_job in handler.get_blocked_jobs():
blocked_job_ids.add(blocked_job.id)
for pending_job in handler.get_pending_jobs():
pending_job_ids.add(pending_job.id)
jobs_with_input = handler.get_jobs_ready_for_input()
num_jobs_with_input += len(jobs_with_input)
for job in jobs_with_input:
job.update_database_with_input(when)
for job in handler.get_jobs_ready_for_first_queue():
job_ids_ready_for_first_queue.append(job.id)
if handler.is_completed() and not handler.recipe.is_completed:
completed_recipe_ids.append(handler.recipe.id)
if handler.recipe.batch_id:
updated_batch_ids.append(handler.recipe.batch_id)
Recipe.objects.complete_recipes(completed_recipe_ids, when)
# Create new messages
self.new_messages.extend(create_blocked_jobs_messages(blocked_job_ids, when))
self.new_messages.extend(create_pending_jobs_messages(pending_job_ids, when))
# Jobs ready for their first queue need to have their input processed
self.new_messages.extend(create_process_job_input_messages(job_ids_ready_for_first_queue))
logger.info('Found %d job(s) that should transition to BLOCKED', len(blocked_job_ids))
logger.info('Found %d job(s) that should transition to PENDING', len(pending_job_ids))
logger.info('Found %d job(s) that received their input', num_jobs_with_input)
logger.info('Found %d job(s) that are ready to be queued', len(job_ids_ready_for_first_queue))
return True
