Exemplo n.º 1
0
    def setUp(self):
        django.setup()

        from product.test import utils as product_test_utils
        self.country = storage_test_utils.create_country()
        self.job_type1 = job_test_utils.create_job_type(name='test1',
                                                        category='test-1',
                                                        is_operational=True)
        self.job1 = job_test_utils.create_job(job_type=self.job_type1)
        self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)

        self.src_file = source_test_utils.create_source(
            data_started='2016-01-01T00:00:00Z',
            data_ended='2016-01-01T00:00:00Z',
            file_name='test.txt',
            is_parsed=True)
        self.product1 = product_test_utils.create_product(
            job_exe=self.job_exe1,
            has_been_published=True,
            is_published=True,
            file_name='test_prod.txt',
            countries=[self.country])
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            descendant=self.product1,
                                            job=self.job1,
                                            job_exe=self.job_exe1)
Exemplo n.º 2
0
    def test_get_source_products(self):
        """Tests calling get_source_products()"""

        from batch.test import utils as batch_test_utils
        from product.test import utils as product_test_utils
        job_exe_1 = job_utils.create_job_exe()
        job_exe_2 = job_utils.create_job_exe()
        product_1 = product_test_utils.create_product(job_exe=job_exe_1,
                                                      has_been_published=True,
                                                      workspace=self.workspace)
        product_2 = product_test_utils.create_product(job_exe=job_exe_2,
                                                      has_been_published=True,
                                                      workspace=self.workspace)
        batch_1 = batch_test_utils.create_batch()
        batch_2 = batch_test_utils.create_batch()
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            descendant=product_1,
                                            job=job_exe_1.job,
                                            job_exe=job_exe_1,
                                            batch=batch_1)
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            descendant=product_2,
                                            job=job_exe_2.job,
                                            job_exe=job_exe_2,
                                            batch=batch_2)

        products = SourceFile.objects.get_source_products(
            self.src_file.id, batch_ids=[batch_1.id])
        self.assertEqual(len(products), 1)
        self.assertEqual(products[0].id, product_1.id)
Exemplo n.º 3
0
    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()

        try:
            import ingest.test.utils as ingest_test_utils
            self.ingest = ingest_test_utils.create_ingest(source_file=self.source)
        except:
            self.ingest = None

        try:
            import product.test.utils as product_test_utils
            self.product1 = product_test_utils.create_product(is_superseded=True)

            product_test_utils.create_file_link(ancestor=self.source, descendant=self.product1)
        except:
            self.product1 = None

        try:
            import product.test.utils as product_test_utils
            self.product2 = product_test_utils.create_product()

            product_test_utils.create_file_link(ancestor=self.source, descendant=self.product2)
        except:
            self.product2 = None
Exemplo n.º 4
0
    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()

        try:
            import ingest.test.utils as ingest_test_utils
            self.ingest = ingest_test_utils.create_ingest(
                source_file=self.source)
        except:
            self.ingest = None

        try:
            import product.test.utils as product_test_utils
            self.product1 = product_test_utils.create_product(
                is_superseded=True)

            product_test_utils.create_file_link(ancestor=self.source,
                                                descendant=self.product1)
        except:
            self.product1 = None

        try:
            import product.test.utils as product_test_utils
            self.product2 = product_test_utils.create_product()

            product_test_utils.create_file_link(ancestor=self.source,
                                                descendant=self.product2)
        except:
            self.product2 = None
Exemplo n.º 5
0
    def setUp(self):
        django.setup()

        from batch.test import utils as batch_test_utils
        from product.test import utils as product_test_utils
        self.country = storage_test_utils.create_country()
        self.src_file = source_test_utils.create_source()
        self.job_type1 = job_test_utils.create_job_type(name='test1', category='test-1', is_operational=True)
        self.job1 = job_test_utils.create_job(job_type=self.job_type1)
        self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
        self.product1 = product_test_utils.create_product(job_exe=self.job_exe1, has_been_published=True,
                                                          is_published=True, file_name='test.txt',
                                                          countries=[self.country])
        product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product1, job=self.job1,
                                            job_exe=self.job_exe1)

        self.batch = batch_test_utils.create_batch()
        self.job_type2 = job_test_utils.create_job_type(name='test2', category='test-2', is_operational=False)
        self.job2 = job_test_utils.create_job(job_type=self.job_type2)
        self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
        self.product2a = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
                                                           is_published=False, countries=[self.country])
        product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2a, job=self.job2,
                                            job_exe=self.job_exe2, batch=self.batch)

        self.product2b = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
                                                           is_published=True, is_superseded=True,
                                                           countries=[self.country])
        product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2b, job=self.job2,
                                            job_exe=self.job_exe2, batch=self.batch)

        self.product2c = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
                                                           is_published=True, countries=[self.country])
        product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2c, job=self.job2,
                                            job_exe=self.job_exe2, batch=self.batch)
Exemplo n.º 6
0
    def test_error_category(self):
        """Tests successfully calling the source jobs view filtered by error category."""

        from product.test import utils as product_test_utils
        error = error_test_utils.create_error(category='DATA')
        job = job_test_utils.create_job(error=error)
        job_exe = job_test_utils.create_job_exe(job=job)
        product_test_utils.create_file_link(ancestor=self.src_file, job=job, job_exe=job_exe)

        url = rest_util.get_url('/sources/%d/jobs/?error_category=%s' % (self.src_file.id, error.category))
        response = self.client.generic('GET', url)
        self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)

        result = json.loads(response.content)
        self.assertEqual(len(result['results']), 1)
        self.assertEqual(result['results'][0]['id'], job.id)
        self.assertEqual(result['results'][0]['error']['category'], error.category)
Exemplo n.º 7
0
    def test_get_source_jobs(self):
        """Tests calling get_source_jobs()"""

        from product.test import utils as product_test_utils
        job_exe = job_utils.create_job_exe()
        product_1 = product_test_utils.create_product(job_exe=job_exe, has_been_published=True,
                                                      workspace=self.workspace)
        product_2 = product_test_utils.create_product(job_exe=job_exe, has_been_published=True,
                                                      workspace=self.workspace)
        product_test_utils.create_file_link(ancestor=self.src_file, descendant=product_1, job=job_exe.job,
                                            job_exe=job_exe)
        product_test_utils.create_file_link(ancestor=self.src_file, descendant=product_2, job=job_exe.job,
                                            job_exe=job_exe)

        jobs = SourceFile.objects.get_source_jobs(self.src_file.id)
        # Should only return one job despite two file_ancestry_link models
        self.assertEqual(len(jobs), 1)
        self.assertEqual(jobs[0].id, job_exe.job.id)
Exemplo n.º 8
0
    def setUp(self):
        django.setup()

        from product.test import utils as product_test_utils
        self.src_file = source_test_utils.create_source()

        self.job_type1 = job_test_utils.create_job_type(
            name='scale-batch-creator', version='1.0', category='test-1')
        self.job1 = job_test_utils.create_job(job_type=self.job_type1,
                                              status='RUNNING')
        self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            job=self.job1,
                                            job_exe=self.job_exe1)

        self.job_type2 = job_test_utils.create_job_type(name='test2',
                                                        version='1.0',
                                                        category='test-2')
        self.job2 = job_test_utils.create_job(job_type=self.job_type2,
                                              status='PENDING')
        self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            job=self.job2,
                                            job_exe=self.job_exe2)

        self.job3 = job_test_utils.create_job(is_superseded=True)
        self.job_exe3 = job_test_utils.create_job_exe(job=self.job3)
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            job=self.job3,
                                            job_exe=self.job_exe3)
Exemplo n.º 9
0
    def test_order_by(self):
        """Tests successfully calling the source jobs view with sorting."""

        from product.test import utils as product_test_utils
        job_type1b = job_test_utils.create_job_type(name='test1',
                                                    version='2.0',
                                                    category='test-1')
        job1b = job_test_utils.create_job(job_type=job_type1b,
                                          status='RUNNING')
        job_exe1b = job_test_utils.create_job_exe(job=job1b)
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            job=job1b,
                                            job_exe=job_exe1b)

        job_type1c = job_test_utils.create_job_type(name='test1',
                                                    version='3.0',
                                                    category='test-1')
        job1c = job_test_utils.create_job(job_type=job_type1c,
                                          status='RUNNING')
        job_exe1c = job_test_utils.create_job_exe(job=job1c)
        product_test_utils.create_file_link(ancestor=self.src_file,
                                            job=job1c,
                                            job_exe=job_exe1c)

        url = rest_util.get_url(
            '/sources/%d/jobs/?order=job_type__name&order=-job_type__version' %
            self.src_file.id)
        response = self.client.generic('GET', url)
        self.assertEqual(response.status_code, status.HTTP_200_OK,
                         response.content)

        result = json.loads(response.content)
        self.assertEqual(len(result['results']), 4)
        self.assertEqual(result['results'][0]['job_type']['id'], job_type1c.id)
        self.assertEqual(result['results'][1]['job_type']['id'], job_type1b.id)
        self.assertEqual(result['results'][2]['job_type']['id'],
                         self.job_type1.id)
        self.assertEqual(result['results'][3]['job_type']['id'],
                         self.job_type2.id)
Exemplo n.º 10
0
    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()
        self.ancestor = product_test_utils.create_product(
            file_name='test_ancestor.txt')
        self.descendant = product_test_utils.create_product(
            file_name='test_descendant.txt')
        self.product = product_test_utils.create_product(
            file_name='test_product.txt')

        product_test_utils.create_file_link(ancestor=self.source,
                                            descendant=self.ancestor)
        product_test_utils.create_file_link(ancestor=self.source,
                                            descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.source,
                                            descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.ancestor,
                                            descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.ancestor,
                                            descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.product,
                                            descendant=self.descendant)
Exemplo n.º 11
0
    def test_execute_ancestor(self):
        """Tests calling DeleteFile.execute() successfully"""

        # Create PurgeResults entry
        source_file = storage_test_utils.create_file()
        trigger = trigger_test_utils.create_trigger_event()
        PurgeResults.objects.create(source_file_id=source_file.id,
                                    trigger_event=trigger)
        self.assertEqual(
            PurgeResults.objects.values_list(
                'num_products_deleted',
                flat=True).get(source_file_id=source_file.id), 0)

        job = job_test_utils.create_job()
        job_exe = job_test_utils.create_job_exe(job=job)

        file_path_1 = os.path.join('my_dir', 'my_file.txt')
        file_path_2 = os.path.join('my_dir', 'my_file1.json')
        file_path_3 = os.path.join('my_dir', 'my_file2.json')
        file_path_4 = os.path.join('my_dir', 'my_file3.json')

        file_1 = storage_test_utils.create_file(file_path=file_path_1,
                                                job_exe=job_exe)
        file_2 = storage_test_utils.create_file(file_path=file_path_2,
                                                job_exe=job_exe)
        file_3 = storage_test_utils.create_file(file_path=file_path_3,
                                                job_exe=job_exe)
        file_4 = storage_test_utils.create_file(file_path=file_path_4,
                                                job_exe=job_exe)
        product_test_utils.create_file_link(ancestor=source_file,
                                            descendant=file_1,
                                            job=job,
                                            job_exe=job_exe)
        product_test_utils.create_file_link(ancestor=source_file,
                                            descendant=file_2,
                                            job=job,
                                            job_exe=job_exe)
        product_test_utils.create_file_link(ancestor=source_file,
                                            descendant=file_3,
                                            job=job,
                                            job_exe=job_exe)
        product_test_utils.create_file_link(ancestor=source_file,
                                            descendant=file_4,
                                            job=job,
                                            job_exe=job_exe)

        # Add files to message
        message = DeleteFiles()
        message.purge = True
        message.job_id = job.id
        message.source_file_id = source_file.id
        message.trigger_id = trigger.id
        if message.can_fit_more():
            message.add_file(file_1.id)
        if message.can_fit_more():
            message.add_file(file_2.id)
        if message.can_fit_more():
            message.add_file(file_3.id)
        if message.can_fit_more():
            message.add_file(file_4.id)

        # Execute message
        result = message.execute()
        self.assertTrue(result)

        # Check results are accurate.
        self.assertEqual(
            PurgeResults.objects.values_list(
                'num_products_deleted', flat=True).get(trigger_event=trigger),
            4)
Exemplo n.º 12
0
    def setUp(self):
        django.setup()

        self.source = source_test_utils.create_source()
        self.ancestor = product_test_utils.create_product(file_name='test_ancestor.txt')
        self.descendant = product_test_utils.create_product(file_name='test_descendant.txt')
        self.product = product_test_utils.create_product(file_name='test_product.txt')

        product_test_utils.create_file_link(ancestor=self.source, descendant=self.ancestor)
        product_test_utils.create_file_link(ancestor=self.source, descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.source, descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.product)
        product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.descendant)
        product_test_utils.create_file_link(ancestor=self.product, descendant=self.descendant)