def __init__(self): """Constructor """ super(Command, self).__init__() self._updater = DatabaseUpdater()
class Command(BaseCommand): """Command that performs a Scale database update """ help = 'Performs a Scale database update' def __init__(self): """Constructor """ super(Command, self).__init__() self._updater = DatabaseUpdater() def handle(self, *args, **options): """See :meth:`django.core.management.base.BaseCommand.handle`. This method starts the command. """ # Register a listener to handle clean shutdowns signal.signal(signal.SIGTERM, self._onsigterm) logger.info('Starting Scale database update') try: self._updater.update() except TerminatedCommand: logger.warning( 'Scale database update stopped, exiting with code %d', SIGTERM_EXIT_CODE) sys.exit(SIGTERM_EXIT_CODE) except: logger.exception( 'Scale database update encountered error, exiting with code %d', GENERAL_FAIL_EXIT_CODE) sys.exit(GENERAL_FAIL_EXIT_CODE) logger.info('Completed Scale database update') def _onsigterm(self, signum, _frame): """See signal callback registration: :py:func:`signal.signal`. This callback performs a clean shutdown when a TERM signal is received. """ logger.info( 'Scale database update received sigterm, telling updater to stop') self._updater.stop()
def test_update_recipe_fields(self): """Tests running the database update to populate new recipe fields in job model""" recipe_1 = recipe_test_utils.create_recipe(is_superseded=True) recipe_2 = recipe_test_utils.create_recipe() recipe_2.root_superseded_recipe = recipe_1 recipe_2.superseded_recipe = recipe_1 recipe_2.completed = now() recipe_2.save() job_1 = job_test_utils.create_job() job_2 = job_test_utils.create_job(is_superseded=True) job_3 = job_test_utils.create_job() recipe_job_1 = recipe_test_utils.create_recipe_job(recipe=recipe_1, job=job_1) recipe_job_2 = recipe_test_utils.create_recipe_job(recipe=recipe_1, job=job_2) recipe_job_3 = recipe_test_utils.create_recipe_job(recipe=recipe_2, job=job_1) recipe_job_3.is_original = False recipe_job_4 = recipe_test_utils.create_recipe_job(recipe=recipe_2, job=job_3) # Run update updater = DatabaseUpdater() updater.update() # Check results job_1 = Job.objects.get(id=job_1.id) self.assertEqual(job_1.recipe_id, recipe_1.id) self.assertEqual(job_1.root_recipe_id, recipe_1.id) job_2 = Job.objects.get(id=job_2.id) self.assertEqual(job_2.recipe_id, recipe_1.id) self.assertEqual(job_2.root_recipe_id, recipe_1.id) job_3 = Job.objects.get(id=job_3.id) self.assertEqual(job_3.recipe_id, recipe_2.id) self.assertEqual(job_3.root_recipe_id, recipe_1.id) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertTrue(recipe_2.is_completed)
def test_update_job_exe_dup(self): """Tests running the database update to remove job execution duplicates""" # Create jobs with duplicate job executions job_type = job_test_utils.create_job_type() job_1 = job_test_utils.create_job(job_type=job_type, num_exes=2) job_2 = job_test_utils.create_job(job_type=job_type, num_exes=3) job_3 = job_test_utils.create_job(job_type=job_type, num_exes=2) # Job 1 job_exe_1 = job_test_utils.create_job_exe(job=job_1, status='COMPLETED', exe_num=1) job_exe_2 = job_test_utils.create_job_exe(job=job_1, status='COMPLETED', exe_num=1) job_exe_3 = job_test_utils.create_job_exe(job=job_1, status='COMPLETED', exe_num=2) job_exe_4 = job_test_utils.create_job_exe(job=job_1, status='COMPLETED', exe_num=2) # Job 2 job_exe_5 = job_test_utils.create_job_exe(job=job_2, status='COMPLETED', exe_num=1) job_exe_6 = job_test_utils.create_job_exe(job=job_2, status='COMPLETED', exe_num=2) job_exe_7 = job_test_utils.create_job_exe(job=job_2, status='COMPLETED', exe_num=2) job_exe_8 = job_test_utils.create_job_exe(job=job_2, status='COMPLETED', exe_num=3) # Job 3 job_exe_9 = job_test_utils.create_job_exe(job=job_3, status='COMPLETED', exe_num=1) # Create some task updates to make sure they get deleted as well task_updates = [] task_updates.append( TaskUpdate(job_exe=job_exe_1, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_1, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_2, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_2, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_3, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_3, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_4, task_id='1234', status='foo')) task_updates.append( TaskUpdate(job_exe=job_exe_4, task_id='1234', status='foo')) TaskUpdate.objects.bulk_create(task_updates) # Run update updater = DatabaseUpdater() updater.update() expected_job_exe_ids = { job_exe_1.id, job_exe_3.id, job_exe_5.id, job_exe_6.id, job_exe_8.id, job_exe_9.id } actual_job_exe_ids = set() for job_exe in JobExecution.objects.all().only('id'): actual_job_exe_ids.add(job_exe.id) self.assertSetEqual(expected_job_exe_ids, actual_job_exe_ids)
def test_update_batch_fields(self): """Tests running the database update to populate new batch fields in job and recipe models""" definition = {"priority": 303} batch_1 = batch_test_utils.create_batch_old(definition=definition) batch_1.recipe_type_rev_id = 1 batch_1.configuration = {} batch_1.save() batch_1.creator_job.status = 'COMPLETED' batch_1.creator_job.save() batch_2 = batch_test_utils.create_batch() recipe_type = recipe_test_utils.create_recipe_type() recipe_1 = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_2 = recipe_test_utils.create_recipe(recipe_type=recipe_type) job_1 = job_test_utils.create_job() job_2 = job_test_utils.create_job() batch_test_utils.create_batch_recipe(batch=batch_1, recipe=recipe_1) batch_test_utils.create_batch_recipe(batch=batch_1, recipe=recipe_2) batch_test_utils.create_batch_job(batch=batch_1, job=job_1) batch_test_utils.create_batch_job(batch=batch_1, job=job_2) batch_1.total_count = 2 batch_1.save() recipe_3 = recipe_test_utils.create_recipe() recipe_4 = recipe_test_utils.create_recipe() job_3 = job_test_utils.create_job() job_4 = job_test_utils.create_job() batch_test_utils.create_batch_recipe(batch=batch_2, recipe=recipe_3) batch_test_utils.create_batch_recipe(batch=batch_2, recipe=recipe_4) batch_test_utils.create_batch_job(batch=batch_2, job=job_3) batch_test_utils.create_batch_job(batch=batch_2, job=job_4) # This batch tests an old batch that never created any recipes time_rev_1 = now() time_rev_2 = time_rev_1 + timedelta(minutes=1) time_batch = time_rev_2 + timedelta(minutes=1) time_rev_3 = time_batch + timedelta(minutes=1) recipe_type_3 = recipe_test_utils.create_recipe_type() recipe_test_utils.edit_recipe_type(recipe_type_3, recipe_type_3.definition) recipe_test_utils.edit_recipe_type(recipe_type_3, recipe_type_3.definition) RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type_3.id, revision_num=1).update(created=time_rev_1) RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type_3.id, revision_num=2).update(created=time_rev_2) RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type_3.id, revision_num=3).update(created=time_rev_3) batch_3 = batch_test_utils.create_batch_old(recipe_type=recipe_type_3) batch_3.recipe_type_rev_id = 1 batch_3.created = time_batch batch_3.save() # Run update updater = DatabaseUpdater() updater.update() # Check results batch_1 = Batch.objects.get(id=batch_1.id) self.assertTrue(batch_1.is_creation_done) self.assertEqual(batch_1.recipes_estimated, 2) recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type.id, recipe_type.revision_num) self.assertEqual(batch_1.recipe_type_rev_id, recipe_type_rev.id) self.assertEqual(batch_1.root_batch_id, batch_1.id) self.assertEqual(batch_1.get_configuration().priority, 303) job_1 = Job.objects.get(id=job_1.id) self.assertEqual(job_1.batch_id, batch_1.id) job_2 = Job.objects.get(id=job_2.id) self.assertEqual(job_2.batch_id, batch_1.id) job_3 = Job.objects.get(id=job_3.id) self.assertEqual(job_3.batch_id, batch_2.id) job_4 = Job.objects.get(id=job_4.id) self.assertEqual(job_4.batch_id, batch_2.id) recipe_1 = Recipe.objects.get(id=recipe_1.id) self.assertEqual(recipe_1.batch_id, batch_1.id) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertEqual(recipe_2.batch_id, batch_1.id) recipe_3 = Recipe.objects.get(id=recipe_3.id) self.assertEqual(recipe_3.batch_id, batch_2.id) recipe_4 = Recipe.objects.get(id=recipe_4.id) self.assertEqual(recipe_4.batch_id, batch_2.id) batch_3 = Batch.objects.get(id=batch_3.id) recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type_3.id, 2) self.assertEqual(batch_3.recipe_type_rev_id, recipe_type_rev.id) self.assertEqual(batch_3.root_batch_id, batch_3.id)