def operation():
JobDropped.create(type=exception.__class__.__name__,
reason=str(exception),
item=item,
source=spider.name,
**response_data)
self.stats.inc_value('monitoring/job_dropped_saved')
def item_dropped(self, item, response, exception, spider):
with db:
JobDropped.create(type=exception.__class__.__name__,
reason=str(exception),
response_url=response.url,
response_backup_path=get_response_backup_path(
response.url),
item=item)
self.stats.inc_value('monitoring/job_dropped_saved')
def test_aggregate_metrics_companies_count_past_jobs_unique(db_connection):
JobDropped.create(type='Expired',
item=dict(company_link='https://example.com/company1'),
reason='...',
response_url='...',
source='...')
JobDropped.create(type='Expired',
item=dict(company_link='https://example.com/company1'),
reason='...',
response_url='...',
source='...')
assert Job.aggregate_metrics()['companies_count'] == 1
def test_aggregate_metrics_rejected_jobs_count(db_connection):
JobDropped.create(source='juniorguru',
type='...',
reason='...',
response_url='...',
item={})
JobDropped.create(source='abc',
type='...',
reason='...',
response_url='...',
item={})
JobDropped.create(source='xyz',
type='...',
reason='...',
response_url='...',
item={})
assert Job.aggregate_metrics()['rejected_jobs_count'] == 2