def test_schedule_job_names(self): """Tests calling BatchManager.schedule_recipes() for a batch that forces all jobs to be re-processed""" handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'job_names': ['Job 1'], } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].batch, batch) self.assertEqual(batch_recipes[0].recipe.recipe_type, self.recipe_type) self.assertEqual(batch_recipes[0].superseded_recipe, handler.recipe) batch_jobs = BatchJob.objects.all() self.assertEqual(len(batch_jobs), 2) for batch_job in batch_jobs: self.assertIn(batch_job.job.job_type, [self.job_type_1, self.job_type_2])
def test_schedule_partial_batch(self, mock_msg_mgr): """Tests calling BatchManager.schedule_recipes() for a batch that is incomplete""" for i in range(5): recipe = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe.save() partials = [] for i in range(5): recipe = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe.is_superseded = True recipe.save() partials.append(recipe) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.created_count, 5) self.assertEqual(batch.total_count, 5) for recipe in partials: recipe.is_superseded = False recipe.save() batch.status = 'SUBMITTED' batch.save() Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED')
def test_schedule_partial_batch(self): """Tests calling BatchManager.schedule_recipes() for a batch that is incomplete""" for i in range(5): Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) partials = [] for i in range(5): handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) handler.recipe.is_superseded = True handler.recipe.save() partials.append(handler.recipe) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) batch = batch_test_utils.create_batch(recipe_type=self.recipe_type) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.created_count, 5) self.assertEqual(batch.total_count, 5) for recipe in partials: recipe.is_superseded = False recipe.save() batch.status = 'SUBMITTED' batch.save() Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 10) self.assertEqual(batch.total_count, 10) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 10)
def test_schedule_date_range_created(self): """Tests calling BatchManager.schedule_recipes() for a batch with a created date range restriction""" recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event).recipe Recipe.objects.filter(pk=recipe1.id).update(created=datetime.datetime(2016, 1, 1)) recipe2 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event).recipe Recipe.objects.filter(pk=recipe2.id).update(created=datetime.datetime(2016, 2, 1)) recipe3 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event).recipe Recipe.objects.filter(pk=recipe3.id).update(created=datetime.datetime(2016, 3, 1)) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'started': '2016-01-10T00:00:00.000Z', 'ended': '2016-02-10T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].superseded_recipe, recipe2)
def test_schedule_date_range_created(self, mock_msg_mgr): """Tests calling BatchManager.schedule_recipes() for a batch with a created date range restriction""" recipe1 = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe1.save() Recipe.objects.filter(pk=recipe1.id).update(created=datetime.datetime(2016, 1, 1, tzinfo=utc)) recipe2 = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe2.save() Recipe.objects.filter(pk=recipe2.id).update(created=datetime.datetime(2016, 2, 1, tzinfo=utc)) recipe3 = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe3.save() Recipe.objects.filter(pk=recipe3.id).update(created=datetime.datetime(2016, 3, 1, tzinfo=utc)) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'started': '2016-01-10T00:00:00.000Z', 'ended': '2016-02-10T00:00:00.000Z', }, } batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1)
def test_schedule_date_range_data_ended(self): """Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction""" file1 = storage_test_utils.create_file() file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc) file1.data_ended = datetime.datetime(2016, 1, 10, tzinfo=utc) file1.save() data1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file1.id, }], 'workspace_id': self.workspace.id, } recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data1), event=self.event).recipe file2 = storage_test_utils.create_file() file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc) file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc) file2.save() data2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file2.id, }], 'workspace_id': self.workspace.id, } Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data2), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'ended': '2016-01-15T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def test_schedule_date_range_data_ended(self): """Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction""" file1 = storage_test_utils.create_file() file1.data_started = datetime.datetime(2016, 1, 1) file1.data_ended = datetime.datetime(2016, 1, 10) file1.save() data1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file1.id, }], 'workspace_id': self.workspace.id, } recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data1), event=self.event).recipe file2 = storage_test_utils.create_file() file2.data_started = datetime.datetime(2016, 2, 1) file2.data_ended = datetime.datetime(2016, 2, 10) file2.save() data2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file2.id, }], 'workspace_id': self.workspace.id, } Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data2), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'ended': '2016-01-15T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def test_schedule_new_batch(self, mock_msg_mgr): """Tests calling BatchManager.schedule_recipes() for a batch that has never been started""" recipe = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe.save() recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1)
def test_schedule_job_names(self, mock_msg_mgr): """Tests calling BatchManager.schedule_recipes() for a batch that forces all jobs to be re-processed""" recipe = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe.save() recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'job_names': ['Job 1'], } batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1)
def test_schedule_new_batch(self): """Tests calling BatchManager.schedule_recipes() for a batch that has never been started""" handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) batch = batch_test_utils.create_batch(recipe_type=self.recipe_type) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].batch, batch) self.assertEqual(batch_recipes[0].recipe.recipe_type, self.recipe_type) self.assertEqual(batch_recipes[0].superseded_recipe, handler.recipe)
def test_schedule_date_range_data_none(self): """Tests calling BatchManager.schedule_recipes() for a batch data date range where no data matches""" Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'started': '2016-01-01T00:00:00.000Z', 'ended': '2016-01-10T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 0) self.assertEqual(batch.total_count, 0)
def test_schedule_date_range_created(self): """Tests calling BatchManager.schedule_recipes() for a batch with a created date range restriction""" recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event).recipe Recipe.objects.filter(pk=recipe1.id).update( created=datetime.datetime(2016, 1, 1, tzinfo=utc)) recipe2 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event).recipe Recipe.objects.filter(pk=recipe2.id).update( created=datetime.datetime(2016, 2, 1, tzinfo=utc)) recipe3 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event).recipe Recipe.objects.filter(pk=recipe3.id).update( created=datetime.datetime(2016, 3, 1, tzinfo=utc)) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'started': '2016-01-10T00:00:00.000Z', 'ended': '2016-02-10T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].superseded_recipe, recipe2)
def test_schedule_partial_batch(self): """Tests calling BatchManager.schedule_recipes() for a batch that is incomplete""" for i in range(5): Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) partials = [] for i in range(5): handler = Recipe.objects.create_recipe( recipe_type=self.recipe_type, data=RecipeData(self.data), event=self.event) handler.recipe.is_superseded = True handler.recipe.save() partials.append(handler.recipe) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) batch = batch_test_utils.create_batch(recipe_type=self.recipe_type) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.created_count, 5) self.assertEqual(batch.total_count, 5) for recipe in partials: recipe.is_superseded = False recipe.save() batch.status = 'SUBMITTED' batch.save() Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 10) self.assertEqual(batch.total_count, 10) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 10)
def test_schedule_date_range_data_none(self): """Tests calling BatchManager.schedule_recipes() for a batch data date range where no data matches""" recipe = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=self.input_data) recipe.save() recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'started': '2016-01-01T00:00:00.000Z', 'ended': '2016-01-10T00:00:00.000Z', }, } batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 0) self.assertEqual(batch.total_count, 0)
def test_update_batch_fields(self): """Tests running the database update to populate new batch fields in job and recipe models""" definition = {"priority": 303} batch_1 = batch_test_utils.create_batch_old(definition=definition) batch_1.recipe_type_rev_id = 1 batch_1.configuration = {} batch_1.save() batch_1.creator_job.status = 'COMPLETED' batch_1.creator_job.save() batch_2 = batch_test_utils.create_batch() recipe_type = recipe_test_utils.create_recipe_type() recipe_1 = recipe_test_utils.create_recipe(recipe_type=recipe_type) recipe_2 = recipe_test_utils.create_recipe(recipe_type=recipe_type) job_1 = job_test_utils.create_job() job_2 = job_test_utils.create_job() batch_test_utils.create_batch_recipe(batch=batch_1, recipe=recipe_1) batch_test_utils.create_batch_recipe(batch=batch_1, recipe=recipe_2) batch_test_utils.create_batch_job(batch=batch_1, job=job_1) batch_test_utils.create_batch_job(batch=batch_1, job=job_2) batch_1.total_count = 2 batch_1.save() recipe_3 = recipe_test_utils.create_recipe() recipe_4 = recipe_test_utils.create_recipe() job_3 = job_test_utils.create_job() job_4 = job_test_utils.create_job() batch_test_utils.create_batch_recipe(batch=batch_2, recipe=recipe_3) batch_test_utils.create_batch_recipe(batch=batch_2, recipe=recipe_4) batch_test_utils.create_batch_job(batch=batch_2, job=job_3) batch_test_utils.create_batch_job(batch=batch_2, job=job_4) # This batch tests an old batch that never created any recipes time_rev_1 = now() time_rev_2 = time_rev_1 + timedelta(minutes=1) time_batch = time_rev_2 + timedelta(minutes=1) time_rev_3 = time_batch + timedelta(minutes=1) recipe_type_3 = recipe_test_utils.create_recipe_type() recipe_test_utils.edit_recipe_type(recipe_type_3, recipe_type_3.definition) recipe_test_utils.edit_recipe_type(recipe_type_3, recipe_type_3.definition) RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type_3.id, revision_num=1).update(created=time_rev_1) RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type_3.id, revision_num=2).update(created=time_rev_2) RecipeTypeRevision.objects.filter( recipe_type_id=recipe_type_3.id, revision_num=3).update(created=time_rev_3) batch_3 = batch_test_utils.create_batch_old(recipe_type=recipe_type_3) batch_3.recipe_type_rev_id = 1 batch_3.created = time_batch batch_3.save() # Run update updater = DatabaseUpdater() updater.update() # Check results batch_1 = Batch.objects.get(id=batch_1.id) self.assertTrue(batch_1.is_creation_done) self.assertEqual(batch_1.recipes_estimated, 2) recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type.id, recipe_type.revision_num) self.assertEqual(batch_1.recipe_type_rev_id, recipe_type_rev.id) self.assertEqual(batch_1.root_batch_id, batch_1.id) self.assertEqual(batch_1.get_configuration().priority, 303) job_1 = Job.objects.get(id=job_1.id) self.assertEqual(job_1.batch_id, batch_1.id) job_2 = Job.objects.get(id=job_2.id) self.assertEqual(job_2.batch_id, batch_1.id) job_3 = Job.objects.get(id=job_3.id) self.assertEqual(job_3.batch_id, batch_2.id) job_4 = Job.objects.get(id=job_4.id) self.assertEqual(job_4.batch_id, batch_2.id) recipe_1 = Recipe.objects.get(id=recipe_1.id) self.assertEqual(recipe_1.batch_id, batch_1.id) recipe_2 = Recipe.objects.get(id=recipe_2.id) self.assertEqual(recipe_2.batch_id, batch_1.id) recipe_3 = Recipe.objects.get(id=recipe_3.id) self.assertEqual(recipe_3.batch_id, batch_2.id) recipe_4 = Recipe.objects.get(id=recipe_4.id) self.assertEqual(recipe_4.batch_id, batch_2.id) batch_3 = Batch.objects.get(id=batch_3.id) recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type_3.id, 2) self.assertEqual(batch_3.recipe_type_rev_id, recipe_type_rev.id) self.assertEqual(batch_3.root_batch_id, batch_3.id)
def test_schedule_date_range_data_full(self, mock_msg_mgr): """Tests calling BatchManager.schedule_recipes() for a batch with a data date range restriction""" file1 = storage_test_utils.create_file() file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc) file1.save() data1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file1.id, }], 'workspace_id': self.workspace.id, } input_data_1 = DataV6(data1).get_data() recipe = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=input_data_1) recipe.save() recipe_file_1 = RecipeInputFile() recipe_file_1.recipe_id = recipe.id recipe_file_1.input_file_id = file1.id recipe_file_1.recipe_input = 'Recipe Input' recipe_file_1.created = recipe.created recipe_file_1.save() file2 = storage_test_utils.create_file() file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc) file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc) file2.save() data2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file2.id, }], 'workspace_id': self.workspace.id, } input_data_2 = DataV6(data2).get_data() recipe2 = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=input_data_2) recipe2.save() recipe_file_2 = RecipeInputFile() recipe_file_2.recipe_id = recipe2.id recipe_file_2.input_file_id = file2.id recipe_file_2.recipe_input = 'Recipe Input' recipe_file_2.created = recipe2.created recipe_file_2.save() file3 = storage_test_utils.create_file() file3.data_ended = datetime.datetime(2016, 3, 1, tzinfo=utc) file3.save() data3 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file3.id, }], 'workspace_id': self.workspace.id, } input_data_3 = DataV6(data3).get_data() recipe3 = Recipe.objects.create_recipe_v6(self.recipe_type_rev, self.event.id, input_data=input_data_3) recipe3.save() recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'started': '2016-02-01T00:00:00.000Z', 'ended': '2016-02-10T00:00:00.000Z', }, } batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1)