def setUp(self):
django.setup()
self.ingest1 = ingest_test_utils.create_ingest(file_name='test1.txt',
status='QUEUED')
self.ingest2 = ingest_test_utils.create_ingest(file_name='test2.txt',
status='INGESTED')
def test_calculate_stats(self):
'''Tests calculating individual statistics for a metrics entry.'''
strike = ingest_test_utils.create_strike()
source_file = source_test_utils.create_source(file_size=200)
ingest_test_utils.create_ingest(strike=strike, source_file=source_file, status='INGESTED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 10),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 1))
ingest_test_utils.create_ingest(strike=strike, status='INGESTED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 20),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 2))
ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 30),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 3))
ingest_test_utils.create_ingest(strike=strike, status='DEFERRED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 40),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 4))
ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 50),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 5))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
entry = entries.first()
self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
self.assertEqual(entry.deferred_count, 1)
self.assertEqual(entry.ingested_count, 2)
self.assertEqual(entry.errored_count, 1)
self.assertEqual(entry.duplicate_count, 1)
self.assertEqual(entry.total_count, 5)
self.assertEqual(entry.file_size_sum, 600)
self.assertEqual(entry.file_size_min, 100)
self.assertEqual(entry.file_size_max, 200)
self.assertEqual(entry.file_size_avg, 120)
self.assertEqual(entry.transfer_time_sum, 9000)
self.assertEqual(entry.transfer_time_min, 600)
self.assertEqual(entry.transfer_time_max, 3000)
self.assertEqual(entry.transfer_time_avg, 1800)
self.assertEqual(entry.ingest_time_sum, 10800)
self.assertEqual(entry.ingest_time_min, 3600)
self.assertEqual(entry.ingest_time_max, 7200)
self.assertEqual(entry.ingest_time_avg, 5400)
def test_calculate_stats(self):
"""Tests calculating individual statistics for a metrics entry."""
strike = ingest_test_utils.create_strike()
source_file = source_test_utils.create_source(file_size=200)
ingest_test_utils.create_ingest(strike=strike, source_file=source_file, status='INGESTED',
transfer_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 10, tzinfo=utc),
ingest_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
ingest_ended=datetime.datetime(2015, 1, 1, 1, tzinfo=utc))
ingest_test_utils.create_ingest(strike=strike, status='INGESTED',
transfer_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 20, tzinfo=utc),
ingest_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
ingest_ended=datetime.datetime(2015, 1, 1, 2, tzinfo=utc))
ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
transfer_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 30, tzinfo=utc),
ingest_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
ingest_ended=datetime.datetime(2015, 1, 1, 3, tzinfo=utc))
ingest_test_utils.create_ingest(strike=strike, status='DEFERRED',
transfer_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 40, tzinfo=utc),
ingest_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
ingest_ended=datetime.datetime(2015, 1, 1, 4, tzinfo=utc))
ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
transfer_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 50, tzinfo=utc),
ingest_started=datetime.datetime(2015, 1, 1, tzinfo=utc),
ingest_ended=datetime.datetime(2015, 1, 1, 5, tzinfo=utc))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
entry = entries.first()
self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
self.assertEqual(entry.deferred_count, 1)
self.assertEqual(entry.ingested_count, 2)
self.assertEqual(entry.errored_count, 1)
self.assertEqual(entry.duplicate_count, 1)
self.assertEqual(entry.total_count, 5)
self.assertEqual(entry.file_size_sum, 600)
self.assertEqual(entry.file_size_min, 100)
self.assertEqual(entry.file_size_max, 200)
self.assertEqual(entry.file_size_avg, 120)
self.assertEqual(entry.transfer_time_sum, 9000)
self.assertEqual(entry.transfer_time_min, 600)
self.assertEqual(entry.transfer_time_max, 3000)
self.assertEqual(entry.transfer_time_avg, 1800)
self.assertEqual(entry.ingest_time_sum, 10800)
self.assertEqual(entry.ingest_time_min, 3600)
self.assertEqual(entry.ingest_time_max, 7200)
self.assertEqual(entry.ingest_time_avg, 5400)
def setUp(self):
django.setup()
self.source_file = source_test_utils.create_source()
from ingest.test import utils as ingest_test_utils
self.strike = ingest_test_utils.create_strike()
self.ingest1 = ingest_test_utils.create_ingest(source_file=self.source_file, status='QUEUED',
strike=self.strike)
self.ingest2 = ingest_test_utils.create_ingest(source_file=self.source_file, status='INGESTED')
def test_calculate_repeated(self):
'''Tests regenerating metrics for a date that already has metrics.'''
strike = ingest_test_utils.create_strike()
ingest_test_utils.create_ingest(strike=strike, status='INGESTED', ingest_ended=datetime.datetime(2015, 1, 1))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
def test_calculate_strike_is_none(self):
"""Tests generating metrics for a date that has ingests with None in Strike field (Scan parent ingest)."""
scan = ingest_test_utils.create_scan()
ingest_test_utils.create_ingest(scan=scan, status='INGESTED',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 0)
def setUp(self):
django.setup()
self.strike = ingest_test_utils.create_strike()
self.ingest1 = ingest_test_utils.create_ingest(file_name='test1.txt', status='QUEUED', strike=self.strike)
self.ingest2 = ingest_test_utils.create_ingest(file_name='test2.txt', status='INGESTED', strike=self.strike)
self.ingest3 = ingest_test_utils.create_ingest(file_name='test3.txt', status='INGESTED', strike=self.strike)
self.ingest4 = ingest_test_utils.create_ingest(file_name='test4.txt', status='INGESTED', strike=self.strike,
data_started=datetime.datetime(2015, 1, 1, tzinfo=timezone.utc),
ingest_ended=datetime.datetime(2015, 2, 1, tzinfo=timezone.utc))
def setUp(self):
django.setup()
self.strike = ingest_test_utils.create_strike()
self.ingest1 = ingest_test_utils.create_ingest(file_name='test1.txt', status='QUEUED', strike=self.strike)
self.ingest2 = ingest_test_utils.create_ingest(file_name='test2.txt', status='INGESTED', strike=self.strike)
self.ingest3 = ingest_test_utils.create_ingest(file_name='test3.txt', status='INGESTED', strike=self.strike)
self.ingest4 = ingest_test_utils.create_ingest(file_name='test4.txt', status='INGESTED', strike=self.strike,
data_started=datetime.datetime(2015, 1, 1, tzinfo=timezone.utc),
ingest_ended=datetime.datetime(2015, 2, 1, tzinfo=timezone.utc))
def test_get_source_ingests(self):
"""Tests calling get_source_ingests()"""
from ingest.test import utils as ingest_test_utils
ingest_1 = ingest_test_utils.create_ingest(source_file=self.src_file, status='INGESTED')
ingest_test_utils.create_ingest(source_file=self.src_file, status='DUPLICATE')
ingests = SourceFile.objects.get_source_ingests(self.src_file.id, statuses=['INGESTED'])
self.assertEqual(len(ingests), 1)
self.assertEqual(ingests[0].id, ingest_1.id)
def test_calculate_repeated(self):
"""Tests regenerating metrics for a date that already has metrics."""
strike = ingest_test_utils.create_strike()
ingest_test_utils.create_ingest(strike=strike, status='INGESTED', ingest_ended=datetime.datetime(2015, 1, 1,
tzinfo=utc))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
def test_multiple_strikes(self):
"""Tests successfully calling the ingest status view with multiple strike process groupings."""
ingest_test_utils.create_strike()
strike3 = ingest_test_utils.create_strike()
ingest_test_utils.create_ingest(file_name='test3.txt', status='INGESTED', strike=strike3)
url = '/ingests/status/'
response = self.client.generic('GET', url)
result = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(result['results']), 3)
def test_multiple_strikes(self):
'''Tests successfully calling the ingest status view with multiple strike process groupings.'''
ingest_test_utils.create_strike()
strike3 = ingest_test_utils.create_strike()
ingest_test_utils.create_ingest(file_name='test3.txt', status='INGESTED', strike=strike3)
url = '/ingests/status/'
response = self.client.generic('GET', url)
result = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(result['results']), 3)
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
try:
import ingest.test.utils as ingest_test_utils
self.ingest = ingest_test_utils.create_ingest(
source_file=self.source)
except:
self.ingest = None
try:
import product.test.utils as product_test_utils
self.product1 = product_test_utils.create_product(
is_superseded=True)
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product1)
except:
self.product1 = None
try:
import product.test.utils as product_test_utils
self.product2 = product_test_utils.create_product()
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product2)
except:
self.product2 = None
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(file_name='my_file.txt')
self.mount = 'host:/path'
self.mount_on = os.path.join('my', 'test')
self.workspace = storage_test_utils.create_workspace()
self.config = StrikeConfiguration({
'version':
'1.0',
'mount':
self.mount,
'transfer_suffix':
'_tmp',
'files_to_ingest': [{
'filename_regex': '.*txt',
'workspace_path': 'foo',
'workspace_name': self.workspace.name,
}],
})
self.job_exe = job_test_utils.create_job_exe()
self.strike_proc = StrikeProcessor(1, self.job_exe.id, self.config)
self.strike_dir = SCALE_INGEST_MOUNT_PATH
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
try:
import ingest.test.utils as ingest_test_utils
self.ingest = ingest_test_utils.create_ingest(source_file=self.source)
except:
self.ingest = None
try:
import product.test.utils as product_test_utils
self.product1 = product_test_utils.create_product(is_superseded=True)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product1)
except:
self.product1 = None
try:
import product.test.utils as product_test_utils
self.product2 = product_test_utils.create_product()
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product2)
except:
self.product2 = None
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(status='INGESTING')
self.job_exe_id = JobExecution.objects.get(job_id=self.ingest.job).id
self.source_file = source_test_utils.create_source(
workspace=self.ingest.workspace)
def test_calculate_stats_partial(self):
"""Tests individual statistics are null when information is unavailable."""
strike = ingest_test_utils.create_strike()
ingest1 = ingest_test_utils.create_ingest(
strike=strike,
status='ERRORED',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
ingest1.file_size = None
ingest1.save()
ingest2 = ingest_test_utils.create_ingest(
strike=strike,
status='DUPLICATE',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
ingest2.file_size = None
ingest2.save()
MetricsIngest.objects.calculate(
datetime.datetime(2015, 1, 1, tzinfo=utc))
entries = MetricsIngest.objects.filter(
occurred=datetime.datetime(2015, 1, 1, tzinfo=utc))
self.assertEqual(len(entries), 1)
entry = entries.first()
self.assertEqual(entry.occurred,
datetime.datetime(2015, 1, 1, tzinfo=utc))
self.assertEqual(entry.deferred_count, 0)
self.assertEqual(entry.ingested_count, 0)
self.assertEqual(entry.errored_count, 1)
self.assertEqual(entry.duplicate_count, 1)
self.assertEqual(entry.total_count, 2)
self.assertIsNone(entry.file_size_sum)
self.assertIsNone(entry.file_size_min)
self.assertIsNone(entry.file_size_max)
self.assertIsNone(entry.file_size_avg)
self.assertIsNone(entry.transfer_time_sum)
self.assertIsNone(entry.transfer_time_min)
self.assertIsNone(entry.transfer_time_max)
self.assertIsNone(entry.transfer_time_avg)
self.assertIsNone(entry.ingest_time_sum)
self.assertIsNone(entry.ingest_time_min)
self.assertIsNone(entry.ingest_time_max)
self.assertIsNone(entry.ingest_time_avg)
def test_successful_manual_kickoff(self, mock_msg_mgr):
"""Tests successfully producing an ingest that immediately calls a recipe"""
ingest = ingest_test_utils.create_ingest(source_file=self.source_file)
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=recipe_test_utils.RECIPE_DEFINITION)
# Call method to test
IngestRecipeHandler().process_manual_ingested_source_file(ingest.id, self.source_file, now(), recipe_type.id)
self.assertEqual(Recipe.objects.all().count(), 1)
self.assertEqual(Recipe.objects.first().recipe_type.name, recipe_type.name)
def test_successful_manual_kickoff(self, mock_create, mock_msg_mgr):
"""Tests successfully producing an ingest that immediately calls a recipe"""
ingest = ingest_test_utils.create_ingest(source_file=self.source_file)
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=recipe_test_utils.RECIPE_DEFINITION)
# Call method to test
IngestRecipeHandler().process_manual_ingested_source_file(
ingest.id, self.source_file, now(), recipe_type.id)
mock_msg_mgr.assert_called_once()
mock_create.assert_called_once()
def test_calculate_stats_partial(self):
'''Tests individual statistics are null when information is unavailable.'''
strike = ingest_test_utils.create_strike()
ingest1 = ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
ingest_ended=datetime.datetime(2015, 1, 1))
ingest1.file_size = None
ingest1.save()
ingest2 = ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
ingest_ended=datetime.datetime(2015, 1, 1))
ingest2.file_size = None
ingest2.save()
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
entry = entries.first()
self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
self.assertEqual(entry.deferred_count, 0)
self.assertEqual(entry.ingested_count, 0)
self.assertEqual(entry.errored_count, 1)
self.assertEqual(entry.duplicate_count, 1)
self.assertEqual(entry.total_count, 2)
self.assertIsNone(entry.file_size_sum)
self.assertIsNone(entry.file_size_min)
self.assertIsNone(entry.file_size_max)
self.assertIsNone(entry.file_size_avg)
self.assertIsNone(entry.transfer_time_sum)
self.assertIsNone(entry.transfer_time_min)
self.assertIsNone(entry.transfer_time_max)
self.assertIsNone(entry.transfer_time_avg)
self.assertIsNone(entry.ingest_time_sum)
self.assertIsNone(entry.ingest_time_min)
self.assertIsNone(entry.ingest_time_max)
self.assertIsNone(entry.ingest_time_avg)
def test_successful_data_time_save(self):
"""Tests calling save_parse_results and checks that the data time is saved within the corresponding ingest
model
"""
from ingest.models import Ingest
from ingest.test import utils
ingest = utils.create_ingest(file_name=self.src_file.file_name, status='INGESTED', source_file=self.src_file)
# Call method to test
SourceFile.objects.save_parse_results(self.src_file.id, FEATURE_COLLECTION_GEOJSON, self.started, self.ended,
[], None)
# Check results
ingest = Ingest.objects.get(pk=ingest.id)
self.assertEqual(ingest.data_started, self.started)
self.assertEqual(ingest.data_ended, self.ended)
def test_successful_data_time_save(self):
"""Tests calling save_parse_results and checks that the data time is saved within the corresponding ingest
model
"""
from ingest.models import Ingest
from ingest.test import utils
ingest = utils.create_ingest(file_name=self.src_file.file_name, status='INGESTED', source_file=self.src_file)
# Call method to test
SourceFile.objects.save_parse_results(self.src_file.id, FEATURE_COLLECTION_GEOJSON, self.started, self.ended,
[], None)
# Check results
ingest = Ingest.objects.get(pk=ingest.id)
self.assertEqual(ingest.data_started, self.started)
self.assertEqual(ingest.data_ended, self.ended)
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(file_name='my_file.txt')
self.mount = 'host:/path'
self.mount_on = os.path.join('my', 'test')
self.workspace = storage_test_utils.create_workspace()
self.config = StrikeConfiguration({
'version': '1.0',
'mount': self.mount,
'transfer_suffix': '_tmp',
'files_to_ingest': [{
'filename_regex': '.*txt',
'workspace_path': 'foo',
'workspace_name': self.workspace.name,
}],
})
self.job_exe = job_test_utils.create_job_exe()
self.strike_proc = StrikeProcessor(1, self.job_exe.id, self.config)
self.strike_dir = SCALE_INGEST_MOUNT_PATH
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(file_name='test1.txt', status='QUEUED')
def test_calculate_filtered(self):
"""Tests generating metrics with only certain ingests."""
ingest_test_utils.create_ingest(status='TRANSFERRING')
ingest_test_utils.create_ingest(status='TRANSFERRED')
ingest_test_utils.create_ingest(status='DEFERRED')
ingest_test_utils.create_ingest(status='QUEUED')
ingest_test_utils.create_ingest(status='INGESTING')
ingest_test_utils.create_ingest(status='INGESTED')
ingest_test_utils.create_ingest(status='ERRORED')
ingest_test_utils.create_ingest(status='DUPLICATE')
ingest_test_utils.create_ingest(strike=ingest_test_utils.create_strike(), status='DEFERRED',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
ingest_test_utils.create_ingest(strike=ingest_test_utils.create_strike(), status='INGESTED',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
ingest_test_utils.create_ingest(strike=ingest_test_utils.create_strike(), status='ERRORED',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
ingest_test_utils.create_ingest(strike=ingest_test_utils.create_strike(), status='DUPLICATE',
ingest_ended=datetime.datetime(2015, 1, 1, tzinfo=utc))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 4)
def test_successful_recipe_kickoff(self, mock_msg_mgr, mock_msg_mgr_rc, mock_msg_mgr_q):
"""Tests successfully producing an ingest that immediately calls a recipe"""
strike_config = {
'version': '7',
'workspace': self.workspace.name,
'monitor': {'type': 'dir-watcher', 'transfer_suffix': '_tmp'},
'files_to_ingest': [{
'filename_regex': 'input_file',
'data_types': ['image_type'],
'new_workspace': self.workspace.name,
'new_file_path': 'my/path'
}],
'recipe': {
'name': self.recipe_v7.name
},
}
config = StrikeConfigurationV6(strike_config).get_configuration()
strike = Strike.objects.create_strike('my_name', 'my_title', 'my_description', config)
ingest = ingest_test_utils.create_ingest(source_file=self.source_file)
# Call method to test
IngestRecipeHandler().process_ingested_source_file(ingest.id, strike, self.source_file, now())
self.assertEqual(Recipe.objects.count(), 1)
self.assertEqual(Recipe.objects.first().recipe_type.name, self.recipe_v7.name)
# Verify ingest event and trigger event objects were created
from ingest.models import IngestEvent
events = IngestEvent.objects.all().values()
self.assertEqual(len(events), 1)
self.assertEqual(events[0]['type'], 'STRIKE')
# Create scan
scan_config = {
'workspace': self.workspace.name,
'scanner': {
'type': 'dir'
},
'files_to_ingest': [{
'filename_regex': 'input_file',
'data_types': ['type1'],
'new_file_path': os.path.join('my', 'path'),
'new_workspace': self.workspace.name,
}],
'recipe': {
'name': self.recipe_v7.name,
},
}
scan_configuration = ScanConfigurationV6(scan_config).get_configuration()
scan = Scan.objects.create_scan('my_name', 'my_title', 'my_description', scan_configuration)
# Call method to test
IngestRecipeHandler().process_ingested_source_file(ingest.id, scan, self.source_file, now())
self.assertEqual(Recipe.objects.count(), 2)
self.assertEqual(Recipe.objects.last().recipe_type.name, self.recipe_v7.name)
# Verify events were created
events = IngestEvent.objects.all().values()
self.assertEqual(len(events), 2)
self.assertEqual(events[1]['type'], 'SCAN')
# Update the recipe then call ingest with revision 1
manifest = job_test_utils.create_seed_manifest(
inputs_files=[{'name': 'INPUT_FILE', 'media_types': ['text/plain'], 'required': True, 'multiple': True}], inputs_json=[])
jt2 = job_test_utils.create_seed_job_type(manifest=manifest)
definition = {'version': '7',
'input': {'files': [{'name': 'INPUT_FILE',
'media_types': ['text/plain'],
'required': True,
'multiple': True}],
'json': []},
'nodes': {'node_a': {'dependencies': [],
'input': {'INPUT_FILE': {'type': 'recipe', 'input': 'INPUT_FILE'}},
'node_type': {'node_type': 'job', 'job_type_name': self.jt1.name,
'job_type_version': self.jt1.version,
'job_type_revision': 1}},
'node_b': {'dependencies': [],
'input': {'INPUT_FILE': {'type': 'recipe', 'input': 'INPUT_FILE'}},
'node_type': {'node_type': 'job', 'job_type_name': jt2.name,
'job_type_version': jt2.version,
'job_type_revision': 1}}}}
recipe_test_utils.edit_recipe_type_v6(recipe_type=self.recipe, definition=definition)
strike_config['recipe'] = {
'name': self.recipe.name,
'revision_num': 1,
}
config = StrikeConfigurationV6(strike_config).get_configuration()
strike = Strike.objects.create_strike('my_name_2', 'my_title_2', 'my_description_2', config)
ingest = ingest_test_utils.create_ingest(source_file=self.source_file)
# Call method to test
IngestRecipeHandler().process_ingested_source_file(ingest.id, strike, self.source_file, now())
self.assertEqual(Recipe.objects.count(), 3)
self.assertEqual(Recipe.objects.first().recipe_type.name, self.recipe.name)
# Verify events were created
events = IngestEvent.objects.all().values()
self.assertEqual(len(events), 3)
self.assertEqual(events[2]['type'], 'STRIKE')
def test_calculate_filtered(self):
'''Tests generating metrics with only certain ingests.'''
ingest_test_utils.create_ingest(status='TRANSFERRING')
ingest_test_utils.create_ingest(status='TRANSFERRED')
ingest_test_utils.create_ingest(status='DEFERRED')
ingest_test_utils.create_ingest(status='QUEUED')
ingest_test_utils.create_ingest(status='INGESTING')
ingest_test_utils.create_ingest(status='INGESTED')
ingest_test_utils.create_ingest(status='ERRORED')
ingest_test_utils.create_ingest(status='DUPLICATE')
ingest_test_utils.create_ingest(status='DEFERRED', ingest_ended=datetime.datetime(2015, 1, 1))
ingest_test_utils.create_ingest(status='INGESTED', ingest_ended=datetime.datetime(2015, 1, 1))
ingest_test_utils.create_ingest(status='ERRORED', ingest_ended=datetime.datetime(2015, 1, 1))
ingest_test_utils.create_ingest(status='DUPLICATE', ingest_ended=datetime.datetime(2015, 1, 1))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 4)
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(status='QUEUED')
self.source_file = source_test_utils.create_source(workspace=self.ingest.workspace)
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(status='QUEUED')
self.source_file = source_test_utils.create_source(
workspace=self.ingest.workspace)
def setUp(self):
django.setup()
self.ingest = ingest_test_utils.create_ingest(status='INGESTING')
self.job_exe_id = JobExecution.objects.get(job_id=self.ingest.job).id
self.source_file = source_test_utils.create_source(workspace=self.ingest.workspace)
