def setUp(self):
django.setup()
from product.test import utils as product_test_utils
self.country = storage_test_utils.create_country()
self.job_type1 = job_test_utils.create_job_type(name='test1',
category='test-1',
is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.src_file = source_test_utils.create_source(
data_started='2016-01-01T00:00:00Z',
data_ended='2016-01-01T00:00:00Z',
file_name='test.txt',
is_parsed=True)
self.product1 = product_test_utils.create_product(
job_exe=self.job_exe1,
has_been_published=True,
is_published=True,
file_name='test_prod.txt',
countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file,
descendant=self.product1,
job=self.job1,
job_exe=self.job_exe1)
def test_get_source_products(self):
"""Tests calling get_source_products()"""
from batch.test import utils as batch_test_utils
from product.test import utils as product_test_utils
job_exe_1 = job_utils.create_job_exe()
job_exe_2 = job_utils.create_job_exe()
product_1 = product_test_utils.create_product(job_exe=job_exe_1,
has_been_published=True,
workspace=self.workspace)
product_2 = product_test_utils.create_product(job_exe=job_exe_2,
has_been_published=True,
workspace=self.workspace)
batch_1 = batch_test_utils.create_batch()
batch_2 = batch_test_utils.create_batch()
product_test_utils.create_file_link(ancestor=self.src_file,
descendant=product_1,
job=job_exe_1.job,
job_exe=job_exe_1,
batch=batch_1)
product_test_utils.create_file_link(ancestor=self.src_file,
descendant=product_2,
job=job_exe_2.job,
job_exe=job_exe_2,
batch=batch_2)
products = SourceFile.objects.get_source_products(
self.src_file.id, batch_ids=[batch_1.id])
self.assertEqual(len(products), 1)
self.assertEqual(products[0].id, product_1.id)
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
try:
import ingest.test.utils as ingest_test_utils
self.ingest = ingest_test_utils.create_ingest(source_file=self.source)
except:
self.ingest = None
try:
import product.test.utils as product_test_utils
self.product1 = product_test_utils.create_product(is_superseded=True)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product1)
except:
self.product1 = None
try:
import product.test.utils as product_test_utils
self.product2 = product_test_utils.create_product()
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product2)
except:
self.product2 = None
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
try:
import ingest.test.utils as ingest_test_utils
self.ingest = ingest_test_utils.create_ingest(
source_file=self.source)
except:
self.ingest = None
try:
import product.test.utils as product_test_utils
self.product1 = product_test_utils.create_product(
is_superseded=True)
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product1)
except:
self.product1 = None
try:
import product.test.utils as product_test_utils
self.product2 = product_test_utils.create_product()
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product2)
except:
self.product2 = None
def setUp(self):
django.setup()
from batch.test import utils as batch_test_utils
from product.test import utils as product_test_utils
self.country = storage_test_utils.create_country()
self.src_file = source_test_utils.create_source()
self.job_type1 = job_test_utils.create_job_type(name='test1', category='test-1', is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.product1 = product_test_utils.create_product(job_exe=self.job_exe1, has_been_published=True,
is_published=True, file_name='test.txt',
countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product1, job=self.job1,
job_exe=self.job_exe1)
self.batch = batch_test_utils.create_batch()
self.job_type2 = job_test_utils.create_job_type(name='test2', category='test-2', is_operational=False)
self.job2 = job_test_utils.create_job(job_type=self.job_type2)
self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
self.product2a = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=False, countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2a, job=self.job2,
job_exe=self.job_exe2, batch=self.batch)
self.product2b = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=True, is_superseded=True,
countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2b, job=self.job2,
job_exe=self.job_exe2, batch=self.batch)
self.product2c = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=True, countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2c, job=self.job2,
job_exe=self.job_exe2, batch=self.batch)
def test_error_category(self):
"""Tests successfully calling the source jobs view filtered by error category."""
from product.test import utils as product_test_utils
error = error_test_utils.create_error(category='DATA')
job = job_test_utils.create_job(error=error)
job_exe = job_test_utils.create_job_exe(job=job)
product_test_utils.create_file_link(ancestor=self.src_file, job=job, job_exe=job_exe)
url = rest_util.get_url('/sources/%d/jobs/?error_category=%s' % (self.src_file.id, error.category))
response = self.client.generic('GET', url)
self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
result = json.loads(response.content)
self.assertEqual(len(result['results']), 1)
self.assertEqual(result['results'][0]['id'], job.id)
self.assertEqual(result['results'][0]['error']['category'], error.category)
def test_get_source_jobs(self):
"""Tests calling get_source_jobs()"""
from product.test import utils as product_test_utils
job_exe = job_utils.create_job_exe()
product_1 = product_test_utils.create_product(job_exe=job_exe, has_been_published=True,
workspace=self.workspace)
product_2 = product_test_utils.create_product(job_exe=job_exe, has_been_published=True,
workspace=self.workspace)
product_test_utils.create_file_link(ancestor=self.src_file, descendant=product_1, job=job_exe.job,
job_exe=job_exe)
product_test_utils.create_file_link(ancestor=self.src_file, descendant=product_2, job=job_exe.job,
job_exe=job_exe)
jobs = SourceFile.objects.get_source_jobs(self.src_file.id)
# Should only return one job despite two file_ancestry_link models
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0].id, job_exe.job.id)
def setUp(self):
django.setup()
from product.test import utils as product_test_utils
self.src_file = source_test_utils.create_source()
self.job_type1 = job_test_utils.create_job_type(
name='scale-batch-creator', version='1.0', category='test-1')
self.job1 = job_test_utils.create_job(job_type=self.job_type1,
status='RUNNING')
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
product_test_utils.create_file_link(ancestor=self.src_file,
job=self.job1,
job_exe=self.job_exe1)
self.job_type2 = job_test_utils.create_job_type(name='test2',
version='1.0',
category='test-2')
self.job2 = job_test_utils.create_job(job_type=self.job_type2,
status='PENDING')
self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
product_test_utils.create_file_link(ancestor=self.src_file,
job=self.job2,
job_exe=self.job_exe2)
self.job3 = job_test_utils.create_job(is_superseded=True)
self.job_exe3 = job_test_utils.create_job_exe(job=self.job3)
product_test_utils.create_file_link(ancestor=self.src_file,
job=self.job3,
job_exe=self.job_exe3)
def test_order_by(self):
"""Tests successfully calling the source jobs view with sorting."""
from product.test import utils as product_test_utils
job_type1b = job_test_utils.create_job_type(name='test1',
version='2.0',
category='test-1')
job1b = job_test_utils.create_job(job_type=job_type1b,
status='RUNNING')
job_exe1b = job_test_utils.create_job_exe(job=job1b)
product_test_utils.create_file_link(ancestor=self.src_file,
job=job1b,
job_exe=job_exe1b)
job_type1c = job_test_utils.create_job_type(name='test1',
version='3.0',
category='test-1')
job1c = job_test_utils.create_job(job_type=job_type1c,
status='RUNNING')
job_exe1c = job_test_utils.create_job_exe(job=job1c)
product_test_utils.create_file_link(ancestor=self.src_file,
job=job1c,
job_exe=job_exe1c)
url = rest_util.get_url(
'/sources/%d/jobs/?order=job_type__name&order=-job_type__version' %
self.src_file.id)
response = self.client.generic('GET', url)
self.assertEqual(response.status_code, status.HTTP_200_OK,
response.content)
result = json.loads(response.content)
self.assertEqual(len(result['results']), 4)
self.assertEqual(result['results'][0]['job_type']['id'], job_type1c.id)
self.assertEqual(result['results'][1]['job_type']['id'], job_type1b.id)
self.assertEqual(result['results'][2]['job_type']['id'],
self.job_type1.id)
self.assertEqual(result['results'][3]['job_type']['id'],
self.job_type2.id)
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
self.ancestor = product_test_utils.create_product(
file_name='test_ancestor.txt')
self.descendant = product_test_utils.create_product(
file_name='test_descendant.txt')
self.product = product_test_utils.create_product(
file_name='test_product.txt')
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.ancestor)
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product)
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.ancestor,
descendant=self.product)
product_test_utils.create_file_link(ancestor=self.ancestor,
descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.product,
descendant=self.descendant)
def test_execute_ancestor(self):
"""Tests calling DeleteFile.execute() successfully"""
# Create PurgeResults entry
source_file = storage_test_utils.create_file()
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger)
self.assertEqual(
PurgeResults.objects.values_list(
'num_products_deleted',
flat=True).get(source_file_id=source_file.id), 0)
job = job_test_utils.create_job()
job_exe = job_test_utils.create_job_exe(job=job)
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file1.json')
file_path_3 = os.path.join('my_dir', 'my_file2.json')
file_path_4 = os.path.join('my_dir', 'my_file3.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1,
job_exe=job_exe)
file_2 = storage_test_utils.create_file(file_path=file_path_2,
job_exe=job_exe)
file_3 = storage_test_utils.create_file(file_path=file_path_3,
job_exe=job_exe)
file_4 = storage_test_utils.create_file(file_path=file_path_4,
job_exe=job_exe)
product_test_utils.create_file_link(ancestor=source_file,
descendant=file_1,
job=job,
job_exe=job_exe)
product_test_utils.create_file_link(ancestor=source_file,
descendant=file_2,
job=job,
job_exe=job_exe)
product_test_utils.create_file_link(ancestor=source_file,
descendant=file_3,
job=job,
job_exe=job_exe)
product_test_utils.create_file_link(ancestor=source_file,
descendant=file_4,
job=job,
job_exe=job_exe)
# Add files to message
message = DeleteFiles()
message.purge = True
message.job_id = job.id
message.source_file_id = source_file.id
message.trigger_id = trigger.id
if message.can_fit_more():
message.add_file(file_1.id)
if message.can_fit_more():
message.add_file(file_2.id)
if message.can_fit_more():
message.add_file(file_3.id)
if message.can_fit_more():
message.add_file(file_4.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Check results are accurate.
self.assertEqual(
PurgeResults.objects.values_list(
'num_products_deleted', flat=True).get(trigger_event=trigger),
4)
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
self.ancestor = product_test_utils.create_product(file_name='test_ancestor.txt')
self.descendant = product_test_utils.create_product(file_name='test_descendant.txt')
self.product = product_test_utils.create_product(file_name='test_product.txt')
product_test_utils.create_file_link(ancestor=self.source, descendant=self.ancestor)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.product)
product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.product, descendant=self.descendant)