def crontabber_status(request):
api = CrontabberState()
# start by assuming the status is OK which means no jobs are broken
context = {'status': 'ALLGOOD'}
all_apps = api.get()['state']
broken = [
name for name, state in all_apps.items()
if state['error_count']
]
blocked = [
name for name, state in all_apps.items()
if set(broken) & set(state['depends_on'])
]
# This infinite loop recurses deeper and deeper into the structure
# to find all jobs that are blocked. If we find that job X is blocked
# by job Y in iteration 1, we need to do another iteration to see if
# there are jobs that are blocked by job X (which was blocked by job Y)
while True:
also_blocked = [
name for name, state in all_apps.items()
if name not in blocked and set(blocked) & set(state['depends_on'])
]
if not also_blocked:
break
blocked += also_blocked
if broken:
# let's change our mind
context['status'] = 'Broken'
context['broken'] = broken
context['blocked'] = blocked
return context
def crontabber_status(request):
api = CrontabberState()
# start by assuming the status is OK which means no jobs are broken
context = {'status': 'ALLGOOD'}
all_apps = api.get()['state']
last_runs = [
x['last_run'] for x in all_apps.values()
if x['last_run']
]
if last_runs:
ancient_times = timezone.now() - datetime.timedelta(
minutes=settings.CRONTABBER_STALE_MINUTES
)
most_recent_run = max(last_runs)
if most_recent_run < ancient_times:
context['status'] = 'Stale'
context['last_run'] = max(last_runs)
else:
# if it's never run, then it's definitely stale
context['status'] = 'Stale'
broken = [
name for name, state in all_apps.items()
if state['error_count']
]
blocked = [
name for name, state in all_apps.items()
if set(broken) & set(state['depends_on'])
]
# This infinite loop recurses deeper and deeper into the structure
# to find all jobs that are blocked. If we find that job X is blocked
# by job Y in iteration 1, we need to do another iteration to see if
# there are jobs that are blocked by job X (which was blocked by job Y)
while True:
also_blocked = [
name for name, state in all_apps.items()
if name not in blocked and set(blocked) & set(state['depends_on'])
]
if not also_blocked:
break
blocked += also_blocked
if broken:
# let's change our mind
context['status'] = 'Broken'
context['broken'] = broken
context['blocked'] = blocked
return context
def test_crontabber_status_not_run_for_a_while(self):
some_time_ago = (
timezone.now() -
datetime.timedelta(minutes=settings.CRONTABBER_STALE_MINUTES))
def mocked_get(**options):
return {
'state': {
'job1': {
'error_count': 0,
'depends_on': [],
'last_run': some_time_ago,
},
'job2': {
'error_count': 0,
'depends_on': ['job1'],
'last_run': some_time_ago,
},
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
data = json.loads(response.content)
eq_(data['status'], 'Stale')
eq_(data['last_run'], some_time_ago.isoformat())
def test_CORS(self):
"""any use of model_wrapper should return a CORS header"""
def mocked_get(**options):
dt = timezone.now()
return {
"state": {
"automatic-emails": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
"ftpscraper": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
}
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('api:model_wrapper', args=('CrontabberState', ))
response = self.client.get(url)
assert response.status_code == 200
assert response['Access-Control-Allow-Origin'] == '*'
def test_CORS(self):
"""any use of model_wrapper should return a CORS header"""
def mocked_get(**options):
dt = timezone.now()
return {
"state": {
"automatic-emails": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
"ftpscraper": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
}
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('api:model_wrapper', args=('CrontabberState',))
response = self.client.get(url)
assert response.status_code == 200
assert response['Access-Control-Allow-Origin'] == '*'
def test_crontabber_status_not_run_for_a_while(self):
some_time_ago = (
timezone.now() - datetime.timedelta(
minutes=settings.CRONTABBER_STALE_MINUTES
)
)
def mocked_get(**options):
return {
'state': {
'job1': {
'error_count': 0,
'depends_on': [],
'last_run': some_time_ago,
},
'job2': {
'error_count': 0,
'depends_on': ['job1'],
'last_run': some_time_ago,
},
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
data = json.loads(response.content)
eq_(data['status'], 'Stale')
eq_(data['last_run'], some_time_ago.isoformat())
def crontabber_status(request):
api = CrontabberState()
# start by assuming the status is OK which means no jobs are broken
context = {'status': 'ALLGOOD'}
all_apps = api.get()['state']
last_runs = [
isodate.parse_datetime(x['last_run']) for x in all_apps.values()
]
if last_runs:
ancient_times = timezone.now() - datetime.timedelta(
minutes=settings.CRONTABBER_STALE_MINUTES)
most_recent_run = max(last_runs)
if most_recent_run < ancient_times:
context['status'] = 'Stale'
context['last_run'] = max(last_runs)
else:
# if it's never run, then it's definitely stale
context['status'] = 'Stale'
broken = [name for name, state in all_apps.items() if state['error_count']]
blocked = [
name for name, state in all_apps.items()
if set(broken) & set(state['depends_on'])
]
# This infinite loop recurses deeper and deeper into the structure
# to find all jobs that are blocked. If we find that job X is blocked
# by job Y in iteration 1, we need to do another iteration to see if
# there are jobs that are blocked by job X (which was blocked by job Y)
while True:
also_blocked = [
name for name, state in all_apps.items()
if name not in blocked and set(blocked) & set(state['depends_on'])
]
if not also_blocked:
break
blocked += also_blocked
if broken:
# let's change our mind
context['status'] = 'Broken'
context['broken'] = broken
context['blocked'] = blocked
return context
def test_crontabber_status_never_run(self):
def mocked_get(**options):
return {'state': {}}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
data = json.loads(response.content)
eq_(data['status'], 'Stale')
def test_crontabber_status_never_run(self):
def mocked_get(**options):
return {
'state': {}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
data = json.loads(response.content)
eq_(data['status'], 'Stale')
def test_crontabber_status_ok(self):
def mocked_get(**options):
recently = timezone.now()
return {
'state': {
'job1': {
'error_count': 0,
'depends_on': [],
'last_run': recently,
}
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
eq_(json.loads(response.content), {'status': 'ALLGOOD'})
def test_crontabber_status_ok(self):
def mocked_get(**options):
recently = timezone.now()
return {
'state': {
'job1': {
'error_count': 0,
'depends_on': [],
'last_run': recently,
}
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
eq_(json.loads(response.content), {'status': 'ALLGOOD'})
def test_crontabber_status_trouble(self):
def mocked_get(**options):
recently = timezone.now()
return {
'state': {
'job1': {
'error_count': 1,
'depends_on': [],
'last_run': recently,
},
'job2': {
'error_count': 0,
'depends_on': ['job1'],
'last_run': recently,
},
'job3': {
'error_count': 0,
'depends_on': ['job2'],
'last_run': recently,
},
'job1b': {
'error_count': 0,
'depends_on': [],
'last_run': recently,
},
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
data = json.loads(response.content)
eq_(data['status'], 'Broken')
eq_(data['broken'], ['job1'])
eq_(data['blocked'], ['job2', 'job3'])
def test_crontabber_status_trouble(self):
def mocked_get(**options):
recently = timezone.now()
return {
'state': {
'job1': {
'error_count': 1,
'depends_on': [],
'last_run': recently,
},
'job2': {
'error_count': 0,
'depends_on': ['job1'],
'last_run': recently,
},
'job3': {
'error_count': 0,
'depends_on': ['job2'],
'last_run': recently,
},
'job1b': {
'error_count': 0,
'depends_on': [],
'last_run': recently,
},
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('monitoring:crontabber_status')
response = self.client.get(url)
eq_(response.status_code, 200)
data = json.loads(response.content)
eq_(data['status'], 'Broken')
eq_(data['broken'], ['job1'])
eq_(data['blocked'], ['job2', 'job3'])
def test_CrontabberState(self):
# The actual dates dont matter, but it matters that it's a
# datetime.datetime object.
def mocked_get(**options):
dt = timezone.now()
return {
"state": {
"automatic-emails": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
"ftpscraper": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
}
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('api:model_wrapper', args=('CrontabberState',))
response = self.client.get(url)
eq_(response.status_code, 200)
dump = json.loads(response.content)
ok_(dump['state'])
def test_CrontabberState(self):
# The actual dates dont matter, but it matters that it's a
# datetime.datetime object.
def mocked_get(**options):
dt = timezone.now()
return {
"state": {
"automatic-emails": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
"ftpscraper": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
}
}
}
CrontabberState.implementation().get.side_effect = mocked_get
url = reverse('api:model_wrapper', args=('CrontabberState', ))
response = self.client.get(url)
assert response.status_code == 200
dump = json.loads(response.content)
assert dump['state']
def test_hit_or_not_hit_ratelimit(self):
def mocked_get(**options):
dt = timezone.now()
return {
"state": {
"automatic-emails": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
"ftpscraper": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
}
}
CrontabberState.implementation().get.side_effect = mocked_get
# doesn't matter much which model we use
url = reverse('api:model_wrapper', args=('CrontabberState', ))
response = self.client.get(url)
assert response.status_code == 200
with self.settings(API_RATE_LIMIT='3/m',
API_RATE_LIMIT_AUTHENTICATED='6/m'):
current_limit = 3 # see above mentioned settings override
# Double to avoid
# https://bugzilla.mozilla.org/show_bug.cgi?id=1148470
for __ in range(current_limit * 2):
response = self.client.get(url,
HTTP_X_FORWARDED_FOR='12.12.12.12')
assert response.status_code == 429
# But it'll work if you use a different X-Forwarded-For IP
# because the rate limit is based on your IP address
response = self.client.get(url, HTTP_X_FORWARDED_FOR='11.11.11.11')
assert response.status_code == 200
user = User.objects.create(username='******')
token = Token.objects.create(user=user,
notes="Just for avoiding rate limit")
response = self.client.get(url, HTTP_AUTH_TOKEN=token.key)
assert response.status_code == 200
for __ in range(current_limit):
response = self.client.get(url)
assert response.status_code == 200
# But even being signed in has a limit.
authenticated_limit = 6 # see above mentioned settings override
assert authenticated_limit > current_limit
for __ in range(authenticated_limit * 2):
response = self.client.get(url)
# Even if you're authenticated - sure the limit is higher -
# eventually you'll run into the limit there too.
assert response.status_code == 429
def test_hit_or_not_hit_ratelimit(self):
def mocked_get(**options):
dt = timezone.now()
return {
"state": {
"automatic-emails": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
"ftpscraper": {
"next_run": dt,
"first_run": dt,
"depends_on": [],
"last_run": dt,
"last_success": dt,
"error_count": 0,
"last_error": {}
},
}
}
CrontabberState.implementation().get.side_effect = mocked_get
# doesn't matter much which model we use
url = reverse('api:model_wrapper', args=('CrontabberState',))
response = self.client.get(url)
eq_(response.status_code, 200)
with self.settings(
API_RATE_LIMIT='3/m',
API_RATE_LIMIT_AUTHENTICATED='6/m'
):
current_limit = 3 # see above mentioned settings override
# Double to avoid
# https://bugzilla.mozilla.org/show_bug.cgi?id=1148470
for __ in range(current_limit * 2):
response = self.client.get(url)
eq_(response.status_code, 429)
# But it'll work if you use a different X-Forwarded-For IP
# because the rate limit is based on your IP address
response = self.client.get(url, HTTP_X_FORWARDED_FOR='11.11.11.11')
eq_(response.status_code, 200)
user = User.objects.create(username='******')
token = Token.objects.create(
user=user,
notes="Just for avoiding rate limit"
)
response = self.client.get(url, HTTP_AUTH_TOKEN=token.key)
eq_(response.status_code, 200)
for __ in range(current_limit):
response = self.client.get(url)
eq_(response.status_code, 200)
# But even being signed in has a limit.
authenticated_limit = 6 # see above mentioned settings override
assert authenticated_limit > current_limit
for __ in range(authenticated_limit * 2):
response = self.client.get(url)
# Even if you're authenticated - sure the limit is higher -
# eventually you'll run into the limit there too.
eq_(response.status_code, 429)