def test_warnings(self):
"""Tests validating a new Strike process where the mount path is changed."""
configuration = {
'mount': 'host:/my/path',
'transfer_suffix': '_tmp',
'files_to_ingest': [{
'filename_regex': '.*txt',
'workspace_path': 'my/path',
'workspace_name': 'raw',
}],
}
ingest_test_utils.create_strike(name='strike-test', configuration=configuration)
url = '/strikes/validation/'
json_data = {
'name': 'strike-test',
'configuration': {
'mount': 'host:/my/new/path',
'transfer_suffix': '_tmp',
'files_to_ingest': [{
'filename_regex': '.*txt',
'workspace_path': 'my/path',
'workspace_name': 'raw',
}],
},
}
response = self.client.generic('POST', url, json.dumps(json_data), 'application/json')
results = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK, response.content)
self.assertEqual(len(results['warnings']), 1)
self.assertEqual(results['warnings'][0]['id'], 'mount_change')
def test_get_metrics_type_choices(self):
'''Tests getting the metrics type with choices.'''
ingest_test_utils.create_strike()
metrics_type = MetricsIngest.objects.get_metrics_type(include_choices=True)
self.assertEqual(metrics_type.name, 'ingests')
self.assertEqual(len(metrics_type.filters), 1)
self.assertEqual(len(metrics_type.choices), 1)
def test_multiple_strikes(self):
"""Tests successfully calling the ingest status view with multiple strike process groupings."""
ingest_test_utils.create_strike()
strike3 = ingest_test_utils.create_strike()
ingest_test_utils.create_ingest(file_name='test3.txt', status='INGESTED', strike=strike3)
url = '/ingests/status/'
response = self.client.generic('GET', url)
result = json.loads(response.content)
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(len(result['results']), 3)
def test_calculate_stats(self):
'''Tests calculating individual statistics for a metrics entry.'''
strike = ingest_test_utils.create_strike()
source_file = source_test_utils.create_source(file_size=200)
ingest_test_utils.create_ingest(strike=strike, source_file=source_file, status='INGESTED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 10),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 1))
ingest_test_utils.create_ingest(strike=strike, status='INGESTED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 20),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 2))
ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 30),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 3))
ingest_test_utils.create_ingest(strike=strike, status='DEFERRED',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 40),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 4))
ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
transfer_started=datetime.datetime(2015, 1, 1),
transfer_ended=datetime.datetime(2015, 1, 1, 0, 50),
ingest_started=datetime.datetime(2015, 1, 1),
ingest_ended=datetime.datetime(2015, 1, 1, 5))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
entry = entries.first()
self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
self.assertEqual(entry.deferred_count, 1)
self.assertEqual(entry.ingested_count, 2)
self.assertEqual(entry.errored_count, 1)
self.assertEqual(entry.duplicate_count, 1)
self.assertEqual(entry.total_count, 5)
self.assertEqual(entry.file_size_sum, 600)
self.assertEqual(entry.file_size_min, 100)
self.assertEqual(entry.file_size_max, 200)
self.assertEqual(entry.file_size_avg, 120)
self.assertEqual(entry.transfer_time_sum, 9000)
self.assertEqual(entry.transfer_time_min, 600)
self.assertEqual(entry.transfer_time_max, 3000)
self.assertEqual(entry.transfer_time_avg, 1800)
self.assertEqual(entry.ingest_time_sum, 10800)
self.assertEqual(entry.ingest_time_min, 3600)
self.assertEqual(entry.ingest_time_max, 7200)
self.assertEqual(entry.ingest_time_avg, 5400)
def test_calculate_repeated(self):
'''Tests regenerating metrics for a date that already has metrics.'''
strike = ingest_test_utils.create_strike()
ingest_test_utils.create_ingest(strike=strike, status='INGESTED', ingest_ended=datetime.datetime(2015, 1, 1))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
def setUp(self):
django.setup()
self.strike = ingest_test_utils.create_strike()
self.ingest1 = ingest_test_utils.create_ingest(file_name='test1.txt', status='QUEUED', strike=self.strike)
self.ingest2 = ingest_test_utils.create_ingest(file_name='test2.txt', status='INGESTED', strike=self.strike)
self.ingest3 = ingest_test_utils.create_ingest(file_name='test3.txt', status='INGESTED', strike=self.strike)
self.ingest4 = ingest_test_utils.create_ingest(file_name='test4.txt', status='INGESTED', strike=self.strike,
data_started=datetime.datetime(2015, 1, 1, tzinfo=timezone.utc),
ingest_ended=datetime.datetime(2015, 2, 1, tzinfo=timezone.utc))
def create_ingest(strike=None, occurred=None, **kwargs):
'''Creates a metrics ingest model for unit testing
:returns: The metrics ingest model
:rtype: :class:`metrics.models.MetricsIngest`
'''
if not strike:
strike = ingest_test_utils.create_strike()
if not occurred:
occurred = timezone.now()
return MetricsIngest.objects.create(strike=strike, occurred=occurred, **kwargs)
def test_get_plot_data_filtered(self):
'''Tests getting the metrics plot data with filters.'''
strike = ingest_test_utils.create_strike()
metrics_test_utils.create_ingest(strike=strike, occurred=datetime.datetime(2015, 1, 1), ingested_count=1)
metrics_test_utils.create_ingest(strike=strike, occurred=datetime.datetime(2015, 1, 20), ingested_count=1)
metrics_test_utils.create_ingest(occurred=datetime.datetime(2015, 1, 1), ingested_count=1)
plot_data = MetricsIngest.objects.get_plot_data(started=datetime.date(2015, 1, 1),
ended=datetime.date(2015, 1, 10),
choice_ids=[strike.id],
columns=[MetricsTypeColumn('ingested_count')])
self.assertEqual(len(plot_data), 1)
self.assertEqual(len(plot_data[0].values), 1)
def test_calculate_stats_partial(self):
'''Tests individual statistics are null when information is unavailable.'''
strike = ingest_test_utils.create_strike()
ingest1 = ingest_test_utils.create_ingest(strike=strike, status='ERRORED',
ingest_ended=datetime.datetime(2015, 1, 1))
ingest1.file_size = None
ingest1.save()
ingest2 = ingest_test_utils.create_ingest(strike=strike, status='DUPLICATE',
ingest_ended=datetime.datetime(2015, 1, 1))
ingest2.file_size = None
ingest2.save()
MetricsIngest.objects.calculate(datetime.date(2015, 1, 1))
entries = MetricsIngest.objects.filter(occurred=datetime.date(2015, 1, 1))
self.assertEqual(len(entries), 1)
entry = entries.first()
self.assertEqual(entry.occurred, datetime.date(2015, 1, 1))
self.assertEqual(entry.deferred_count, 0)
self.assertEqual(entry.ingested_count, 0)
self.assertEqual(entry.errored_count, 1)
self.assertEqual(entry.duplicate_count, 1)
self.assertEqual(entry.total_count, 2)
self.assertIsNone(entry.file_size_sum)
self.assertIsNone(entry.file_size_min)
self.assertIsNone(entry.file_size_max)
self.assertIsNone(entry.file_size_avg)
self.assertIsNone(entry.transfer_time_sum)
self.assertIsNone(entry.transfer_time_min)
self.assertIsNone(entry.transfer_time_max)
self.assertIsNone(entry.transfer_time_avg)
self.assertIsNone(entry.ingest_time_sum)
self.assertIsNone(entry.ingest_time_min)
self.assertIsNone(entry.ingest_time_max)
self.assertIsNone(entry.ingest_time_avg)
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace(name='raw')
self.strike = ingest_test_utils.create_strike()
def setUp(self):
django.setup()
self.strike1 = ingest_test_utils.create_strike(name='test-1', description='test A')
self.strike2 = ingest_test_utils.create_strike(name='test-2', description='test Z')
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace(name='raw')
self.strike = ingest_test_utils.create_strike()
def setUp(self):
django.setup()
self.strike1 = ingest_test_utils.create_strike(name='test-1', description='test A')
self.strike2 = ingest_test_utils.create_strike(name='test-2', description='test Z')