def test_with_valid_data(self, fake_requests_obj):
"""Test fetch_bugs valid bug data processing."""
if (not getattr(settings, 'REMOZILLA_USERNAME', None) or
not getattr(settings, 'REMOZILLA_PASSWORD', None)):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
previous_last_updated_time = get_last_updated_date()
request = requests.Request()
request.status_code = 200
bug_data = {'bugs': [{'id': 7788,
'summary':'This is summary',
'creator': {'name': '*****@*****.**'},
'creation_time': '2010-10-5T13:45:23Z',
'component':'Swag Requests',
'whiteboard':'This is whiteboard',
'cc': [{'name': '*****@*****.**'},
{'name': '*****@*****.**'}],
'assigned_to':{'name':'*****@*****.**'},
'status':'resolved',
'resolution':'invalid',
'cf_due_date': None},
{'id': 1199,
'summary':'New summary',
'creator': {'name': '*****@*****.**'},
'creation_time': '2012-12-5T11:30:23Z',
'component':'Budget Requests',
'cc': [{'name': '*****@*****.**'},
{'name': '*****@*****.**'}],
'assigned_to':{'name':'*****@*****.**'},
'status':'resolved',
'resolution':'invalid',
'cf_due_date': None}]}
request.text = json.dumps(bug_data)
(fake_requests_obj.expects_call().returns(request))
fetch_bugs()
eq_(Bug.objects.all().count(), len(bug_data['bugs']))
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.due_date, None)
eq_(bug.resolution, 'INVALID')
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
now = timezone.now()
if not days:
days = (now - get_last_updated_date()).days + 1
for component in COMPONENTS:
url = URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timedelta=days)
response = requests.get(url)
if response.status_code != 200:
raise ValueError('Invalid response from server.')
bugs = json.loads(response.text)
for bdata in bugs['bugs']:
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
bug.creator = get_object_or_none(User,
email=bdata['creator']['name'])
bug.bug_creation_time = datetime.strptime(bdata['creation_time'],
'%Y-%m-%dT%H:%M:%SZ')
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for person in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=person['name'])
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(User,
email=(bdata['assigned_to']\
['name']))
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.due_date = bdata.get('cf_due_date', None)
if 'last_change_time' in bdata:
bug.bug_last_change_time = datetime.strptime(
bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ')
bug.save()
set_last_updated_date(now)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
now = timezone.now()
if not days:
days = (now - get_last_updated_date()).days + 1
for component in COMPONENTS:
url = URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timedelta=days)
response = requests.get(url)
if response.status_code != 200:
raise ValueError('Invalid response from server.')
bugs = json.loads(response.text)
for bdata in bugs['bugs']:
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
bug.creator = get_object_or_none(User,
email=bdata['creator']['name'])
bug.bug_creation_time = datetime.strptime(bdata['creation_time'],
'%Y-%m-%dT%H:%M:%SZ')
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for person in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=person['name'])
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(User,
email=(bdata['assigned_to']\
['name']))
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.due_date = bdata.get('cf_due_date', None)
if 'last_change_time' in bdata:
bug.bug_last_change_time = datetime.strptime(
bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ')
bug.save()
set_last_updated_date(now)
def test_with_valid_data(self, fake_get, switch_is_active_mock):
"""Test fetch_bugs valid bug data processing."""
UserFactory.create(username='******')
if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or
not getattr(settings, 'REMOZILLA_PASSWORD', None))):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
switch_is_active_mock.return_value = True
previous_last_updated_time = get_last_updated_date()
first_request = requests.Request()
first_request.status_code = 200
user = UserFactory.create(groups=['Rep'], email='*****@*****.**')
bug_data = {'bugs': [{'id': 7788,
'summary': 'This is summary',
'creator': {'name': '*****@*****.**'},
'creation_time': '2010-10-5T13:45:23Z',
'component': 'Budget Requests',
'whiteboard': 'This is whiteboard',
'cc': [{'name': '*****@*****.**'},
{'name': '*****@*****.**'}],
'assigned_to': {'name': '*****@*****.**'},
'status': 'resolved',
'flags': [
{'status': '?',
'name': 'remo-approval'},
{'status': '?',
'name': 'needinfo',
'requestee': {
'name': settings.REPS_COUNCIL_ALIAS}},
{'status': '?',
'name': 'needinfo',
'requestee': {
'name': '*****@*****.**'}}
],
'resolution': 'invalid'},
{'id': 1199,
'summary': 'New summary',
'creator': {'name': '*****@*****.**'},
'creation_time': '2012-12-5T11:30:23Z',
'component': 'Budget Requests',
'whiteboard': 'Council Reviewer Assigned',
'cc': [{'name': '*****@*****.**'},
{'name': '*****@*****.**'}],
'flags': [{'status': '?',
'name': 'remo-approval'},
{'status': '?',
'name': 'remo-review'}],
'assigned_to': {'name': '*****@*****.**'},
'status': 'resolved',
'resolution': 'invalid'}]}
first_request.text = json.dumps(bug_data)
second_request = requests.Request()
second_request.status_code = 200
bug_data['bugs'][0]['component'] = 'Planning'
bug_data['bugs'][0]['id'] = 7789
bug_data['bugs'][1]['component'] = 'Planning'
bug_data['bugs'][1]['id'] = 1200
second_request.text = json.dumps(bug_data)
empty_request = requests.Request()
empty_request.status_code = 200
empty_request.text = json.dumps({'bugs': []})
values = [first_request, empty_request, second_request, empty_request]
fake_get.side_effect = values
fetch_bugs(components=['Planning', 'Budget Requests'])
eq_(Bug.objects.all().count(), 4)
eq_(Bug.objects.filter(component='Planning').count(), 2)
eq_(Bug.objects.filter(component='Budget Requests').count(), 2)
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.resolution, 'INVALID')
ok_(bug.council_vote_requested)
ok_(user in bug.budget_needinfo.all())
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
ok_(not bug.council_vote_requested)
ok_(bug.pending_mentor_validation)
ok_(bug.council_member_assigned)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
login_url = LOGIN_URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD)
response = requests.get(login_url).json()
error = response.get('error')
# Check the server response and get the token
if error:
raise ValueError('Invalid response from server, {0}.'.format(
response['error']))
token = response['token']
now = timezone.now()
if not days:
changed_date = get_last_updated_date()
else:
changed_date = now - timedelta(int(days))
for component in components:
offset = 0
url = URL.format(token=token,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timestamp=changed_date,
offset=offset,
limit=LIMIT)
while True:
bugs = requests.get(url).json()
error = bugs.get('error')
# Check the server response and get the token
if error:
raise ValueError('Invalid response from server, {0}.'.format(
bugs['message']))
remo_bugs = bugs.get('bugs', [])
if not remo_bugs:
break
for bdata in remo_bugs:
# Get comments for current bug
comment_url = COMMENT_URL.format(id=bdata['id'], token=token)
comments = requests.get(comment_url).json()
error = comments.get('error')
if error:
raise ValueError(
'Invalid response from server, {0}.'.format(
comments['message']))
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = unicode(bdata.get('summary', ''))
creator_email = bdata['creator']
bug.creator = get_object_or_none(User, email=creator_email)
bug.bug_creation_time = (parse_bugzilla_time(
bdata['creation_time']))
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for email in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=email)
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(
User, email=bdata['assigned_to'])
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.bug_last_change_time = parse_bugzilla_time(
bdata.get('last_change_time'))
automated_voting_trigger = 0
bug.budget_needinfo.clear()
bug.council_member_assigned = False
bug.pending_mentor_validation = False
for flag in bdata.get('flags', []):
if ((flag['status'] == '?'
and flag['name'] == 'remo-approval')):
automated_voting_trigger += 1
if 'Council Reviewer Assigned' in bug.whiteboard:
bug.council_member_assigned = True
if ((flag['status'] == '?' and flag['name'] == 'needinfo'
and 'requestee' in flag and flag['requestee']
== (settings.REPS_COUNCIL_ALIAS))):
automated_voting_trigger += 1
if flag['status'] == '?' and flag['name'] == 'remo-review':
bug.pending_mentor_validation = True
if (flag['status'] == '?' and flag['name'] == 'needinfo'
and 'requestee' in flag):
email = flag['requestee']
user = get_object_or_none(User, email=email)
if user:
bug.budget_needinfo.add(user)
if ((automated_voting_trigger == 2
and waffle.switch_is_active('automated_polls'))):
bug.council_vote_requested = True
unicode_id = unicode(bdata['id'])
bug_comments = comments['bugs'][unicode_id]['comments']
if bug_comments and bug_comments[0].get('text', ''):
# Enforce unicode encoding.
bug.first_comment = unicode(bug_comments[0]['text'])
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
def test_with_valid_data(self, mocked_request, switch_is_active_mock):
"""Test fetch_bugs valid bug data processing."""
UserFactory.create(username='******')
if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or
not getattr(settings, 'REMOZILLA_PASSWORD', None))):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
switch_is_active_mock.return_value = True
previous_last_updated_time = get_last_updated_date()
mentor = UserFactory.create()
user = UserFactory.create(groups=['Rep'], email='*****@*****.**',
userprofile__mentor=mentor)
login_data = {u'token': u'bugzilla_token',
u'id': 12345}
bug_data = [{'id': 7788,
'summary': 'This is summary',
'creator': '*****@*****.**',
'creation_time': '2010-10-5T13:45:23Z',
'component': 'Budget Requests',
'whiteboard': 'This is whiteboard',
'cc': ['*****@*****.**', '*****@*****.**'],
'assigned_to': '*****@*****.**',
'status': 'resolved',
'flags': [{'status': '?',
'name': 'remo-approval'},
{'status': '?',
'name': 'needinfo',
'requestee': settings.REPS_REVIEW_ALIAS},
{'status': '?',
'name': 'needinfo',
'requestee': '*****@*****.**'}],
'resolution': 'invalid'},
{'id': 1199,
'summary': 'New summary',
'creator': '*****@*****.**',
'creation_time': '2012-12-5T11:30:23Z',
'component': 'Budget Requests',
'whiteboard': 'Review Team approval needed',
'cc': ['*****@*****.**', '*****@*****.**'],
'flags': [{'status': '?',
'name': 'remo-approval'},
{'status': '?',
'name': 'remo-review'}],
'assigned_to': '*****@*****.**',
'status': 'resolved',
'resolution': 'invalid'}]
def mocked_get(url, *args, **kwargs):
mocked_response = Mock()
if 'login' in url:
mocked_response.json.return_value = login_data
return mocked_response
elif 'comment' in url:
comments = {
'bugs': {
'7788': {
'comments': [{'text': 'foo'}, {'text': 'bar'}]
},
'1199': {
'comments': [{'text': 'bar'}, {'text': 'foo'}]
}
}
}
mocked_response.json.return_value = comments
return mocked_response
else:
mocked_response.json.return_value = {'bugs': bug_data}
url_params = parse_qs(urlparse(url).query)
offset = url_params.get('offset')
if offset and int(offset[0]) > 0:
mocked_response.json.return_value = {'bugs': []}
return mocked_response
mocked_request.side_effect = mocked_get
fetch_bugs()
eq_(Bug.objects.all().count(), 2)
eq_(Bug.objects.filter(component='Budget Requests').count(), 2)
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.resolution, 'INVALID')
eq_(bug.creator, User.objects.get(email='*****@*****.**'))
ok_(bug.council_vote_requested)
ok_(user in bug.budget_needinfo.all())
eq_(bug.first_comment, 'foo')
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
ok_(not bug.council_vote_requested)
ok_(bug.pending_mentor_validation)
ok_(bug.council_member_assigned)
eq_(bug.first_comment, 'bar')
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
now = timezone.now()
if not days:
days = (now - get_last_updated_date()).days + 1
for component in components:
offset = 0
url = URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timedelta=days, offset=offset, limit=LIMIT)
while True:
response = requests.get(url)
if response.status_code != 200:
raise ValueError('Invalid response from server.')
bugs = json.loads(response.text)
if not bugs['bugs']:
break
for bdata in bugs['bugs']:
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
creator_name = bdata['creator']['name']
bug.creator = get_object_or_none(User, email=creator_name)
creation_time = datetime.strptime(bdata['creation_time'],
'%Y-%m-%dT%H:%M:%SZ')
bug.bug_creation_time = creation_time
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for person in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=person['name'])
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = (
get_object_or_none(User,
email=(bdata['assigned_to']['name'])))
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.due_date = bdata.get('cf_due_date', None)
if 'last_change_time' in bdata:
bug.bug_last_change_time = datetime.strptime(
bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ')
flags = bdata.get('flags', [])
bug.flag_status = next((item['status'] for item in flags
if item['status'] == '?'), '')
bug.flag_name = next((item['name'] for item in flags
if item['name'] == 'remo-review'), '')
comments = bdata.get('comments', [])
if comments and comments[0].get('text', ''):
bug.first_comment = comments[0]['text']
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
def test_with_valid_data(self, mocked_request, switch_is_active_mock):
"""Test fetch_bugs valid bug data processing."""
UserFactory.create(username='******')
if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or
not getattr(settings, 'REMOZILLA_PASSWORD', None))):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
switch_is_active_mock.return_value = True
previous_last_updated_time = get_last_updated_date()
mentor = UserFactory.create()
user = UserFactory.create(groups=['Rep'], email='*****@*****.**',
userprofile__mentor=mentor)
login_data = {u'token': u'bugzilla_token',
u'id': 12345}
bug_data = [{'id': 7788,
'summary': 'This is summary',
'creator': '*****@*****.**',
'creation_time': '2010-10-5T13:45:23Z',
'component': 'Budget Requests',
'whiteboard': 'This is whiteboard',
'cc': ['*****@*****.**', '*****@*****.**'],
'assigned_to': '*****@*****.**',
'status': 'resolved',
'flags': [{'status': '?',
'name': 'remo-approval'},
{'status': '?',
'name': 'needinfo',
'requestee': settings.REPS_COUNCIL_ALIAS},
{'status': '?',
'name': 'needinfo',
'requestee': '*****@*****.**'}],
'resolution': 'invalid'},
{'id': 1199,
'summary': 'New summary',
'creator': '*****@*****.**',
'creation_time': '2012-12-5T11:30:23Z',
'component': 'Budget Requests',
'whiteboard': 'Council Reviewer Assigned',
'cc': ['*****@*****.**', '*****@*****.**'],
'flags': [{'status': '?',
'name': 'remo-approval'},
{'status': '?',
'name': 'remo-review'}],
'assigned_to': '*****@*****.**',
'status': 'resolved',
'resolution': 'invalid'}]
def mocked_get(url, *args, **kwargs):
mocked_response = Mock()
if 'login' in url:
mocked_response.json.return_value = login_data
return mocked_response
elif 'comment' in url:
comments = {
'bugs': {
'7788': {
'comments': [{'text': 'foo'}, {'text': 'bar'}]
},
'1199': {
'comments': [{'text': 'bar'}, {'text': 'foo'}]
}
}
}
mocked_response.json.return_value = comments
return mocked_response
else:
mocked_response.json.return_value = {'bugs': bug_data}
url_params = parse_qs(urlparse(url).query)
offset = url_params.get('offset')
if offset and int(offset[0]) > 0:
mocked_response.json.return_value = {'bugs': []}
return mocked_response
mocked_request.side_effect = mocked_get
fetch_bugs()
eq_(Bug.objects.all().count(), 2)
eq_(Bug.objects.filter(component='Budget Requests').count(), 2)
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.resolution, 'INVALID')
eq_(bug.creator, User.objects.get(email='*****@*****.**'))
ok_(bug.council_vote_requested)
ok_(user in bug.budget_needinfo.all())
eq_(bug.first_comment, 'foo')
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
ok_(not bug.council_vote_requested)
ok_(bug.pending_mentor_validation)
ok_(bug.council_member_assigned)
eq_(bug.first_comment, 'bar')
def test_with_valid_data(self, fake_requests_obj):
"""Test fetch_bugs valid bug data processing."""
if (not getattr(settings, 'REMOZILLA_USERNAME', None)
or not getattr(settings, 'REMOZILLA_PASSWORD', None)):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
previous_last_updated_time = get_last_updated_date()
first_request = requests.Request()
first_request.status_code = 200
bug_data = {
'bugs': [{
'id':
7788,
'summary':
'This is summary',
'creator': {
'name': '*****@*****.**'
},
'creation_time':
'2010-10-5T13:45:23Z',
'component':
'Swag Requests',
'whiteboard':
'This is whiteboard',
'cc': [{
'name': '*****@*****.**'
}, {
'name': '*****@*****.**'
}],
'assigned_to': {
'name': '*****@*****.**'
},
'status':
'resolved',
'resolution':
'invalid',
'cf_due_date':
None
}, {
'id':
1199,
'summary':
'New summary',
'creator': {
'name': '*****@*****.**'
},
'creation_time':
'2012-12-5T11:30:23Z',
'component':
'Swag requests',
'cc': [{
'name': '*****@*****.**'
}, {
'name': '*****@*****.**'
}],
'assigned_to': {
'name': '*****@*****.**'
},
'status':
'resolved',
'resolution':
'invalid',
'cf_due_date':
None
}]
}
first_request.text = json.dumps(bug_data)
second_request = requests.Request()
second_request.status_code = 200
bug_data['bugs'][0]['component'] = 'Planning'
bug_data['bugs'][0]['id'] = 7789
bug_data['bugs'][1]['component'] = 'Planning'
bug_data['bugs'][1]['id'] = 1200
second_request.text = json.dumps(bug_data)
empty_request = requests.Request()
empty_request.status_code = 200
empty_request.text = json.dumps({'bugs': []})
fake_requests_obj.expects_call()
fake_requests_obj.returns(first_request)
fake_requests_obj.next_call().returns(empty_request)
fake_requests_obj.next_call().returns(second_request)
fake_requests_obj.next_call().returns(empty_request)
fetch_bugs(components=['Planning', 'Swag Requests'])
eq_(Bug.objects.all().count(), 4)
eq_(Bug.objects.filter(component='Planning').count(), 2)
eq_(Bug.objects.filter(component='Swag Requests').count(), 2)
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.due_date, None)
eq_(bug.resolution, 'INVALID')
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
now = timezone.now()
if not days:
days = (now - get_last_updated_date()).days + 1
for component in components:
offset = 0
url = URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timedelta=days, offset=offset, limit=LIMIT)
while True:
response = requests.get(url)
if response.status_code != 200:
raise ValueError('Invalid response from server.')
bugs = json.loads(response.text)
if not bugs['bugs']:
break
for bdata in bugs['bugs']:
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
creator_name = bdata['creator']['name']
bug.creator = get_object_or_none(User, email=creator_name)
bug.bug_creation_time = (
parse_bugzilla_time(bdata['creation_time']))
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for person in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=person['name'])
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(
User, email=bdata['assigned_to']['name'])
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.bug_last_change_time = parse_bugzilla_time(
bdata.get('last_change_time'))
automated_voting_trigger = 0
for flag in bdata.get('flags', []):
if ((flag['status'] == '?'
and flag['name'] == 'remo-approval')):
automated_voting_trigger += 1
if ((flag['status'] == '?'
and flag['name'] == 'needinfo'
and 'requestee' in flag
and flag['requestee']['name'] == (
settings.REPS_COUNCIL_ALIAS))):
automated_voting_trigger += 1
if ((automated_voting_trigger == 2
and waffle.switch_is_active('automated_polls'))):
bug.council_vote_requested = True
comments = bdata.get('comments', [])
if comments and comments[0].get('text', ''):
bug.first_comment = comments[0]['text']
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
def test_with_valid_data(self, fake_requests_obj):
"""Test fetch_bugs valid bug data processing."""
if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or
not getattr(settings, 'REMOZILLA_PASSWORD', None))):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
previous_last_updated_time = get_last_updated_date()
first_request = requests.Request()
first_request.status_code = 200
bug_data = {'bugs': [{'id': 7788,
'summary': 'This is summary',
'creator': {'name': '*****@*****.**'},
'creation_time': '2010-10-5T13:45:23Z',
'component': 'Swag Requests',
'whiteboard': 'This is whiteboard',
'cc': [{'name': '*****@*****.**'},
{'name': '*****@*****.**'}],
'assigned_to': {'name': '*****@*****.**'},
'status': 'resolved',
'flags': [
{'requestee': {
'ref': '',
'name': '*****@*****.**'},
'status': '?',
'name': 'remo-review'}],
'resolution': 'invalid'},
{'id': 1199,
'summary': 'New summary',
'creator': {'name': '*****@*****.**'},
'creation_time': '2012-12-5T11:30:23Z',
'component': 'Swag requests',
'cc': [{'name': '*****@*****.**'},
{'name': '*****@*****.**'}],
'assigned_to': {'name': '*****@*****.**'},
'status': 'resolved',
'resolution': 'invalid'}]}
first_request.text = json.dumps(bug_data)
second_request = requests.Request()
second_request.status_code = 200
bug_data['bugs'][0]['component'] = 'Planning'
bug_data['bugs'][0]['id'] = 7789
bug_data['bugs'][1]['component'] = 'Planning'
bug_data['bugs'][1]['id'] = 1200
second_request.text = json.dumps(bug_data)
empty_request = requests.Request()
empty_request.status_code = 200
empty_request.text = json.dumps({'bugs': []})
fake_requests_obj.expects_call()
fake_requests_obj.returns(first_request)
fake_requests_obj.next_call().returns(empty_request)
fake_requests_obj.next_call().returns(second_request)
fake_requests_obj.next_call().returns(empty_request)
fetch_bugs(components=['Planning', 'Swag Requests'])
eq_(Bug.objects.all().count(), 4)
eq_(Bug.objects.filter(component='Planning').count(), 2)
eq_(Bug.objects.filter(component='Swag Requests').count(), 2)
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.resolution, 'INVALID')
eq_(bug.council_vote_requested, True)
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
eq_(bug.council_vote_requested, False)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
now = timezone.now()
if not days:
days = (now - get_last_updated_date()).days + 1
for component in components:
offset = 0
url = URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timedelta=days, offset=offset, limit=LIMIT)
while True:
response = requests.get(url)
if response.status_code != 200:
raise ValueError('Invalid response from server.')
bugs = json.loads(response.text)
if not bugs['bugs']:
break
for bdata in bugs['bugs']:
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
creator_name = bdata['creator']['name']
bug.creator = get_object_or_none(User, email=creator_name)
bug.bug_creation_time = (
parse_bugzilla_time(bdata['creation_time']))
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for person in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=person['name'])
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(
User, email=bdata['assigned_to']['name'])
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.bug_last_change_time = parse_bugzilla_time(
bdata.get('last_change_time'))
automated_voting_trigger = 0
bug.budget_needinfo.clear()
bug.council_member_assigned = False
bug.pending_mentor_validation = False
for flag in bdata.get('flags', []):
if ((flag['status'] == '?'
and flag['name'] == 'remo-approval')):
automated_voting_trigger += 1
if 'Council Reviewer Assigned' in bug.whiteboard:
bug.council_member_assigned = True
if ((flag['status'] == '?'
and flag['name'] == 'needinfo'
and 'requestee' in flag
and flag['requestee']['name'] == (
settings.REPS_COUNCIL_ALIAS))):
automated_voting_trigger += 1
if ((flag['status'] == '?'
and flag['name'] == 'remo-review')):
bug.pending_mentor_validation = True
if ((flag['status'] == '?'
and flag['name'] == 'needinfo'
and 'requestee' in flag
and bug.component == 'Budget Requests')):
email = flag['requestee']['name']
user = get_object_or_none(User, email=email)
if user:
bug.budget_needinfo.add(user)
if ((automated_voting_trigger == 2
and waffle.switch_is_active('automated_polls'))):
bug.council_vote_requested = True
comments = bdata.get('comments', [])
if comments and comments[0].get('text', ''):
bug.first_comment = comments[0]['text']
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
# TODO: This can trigger a does not exist error because the task was picked
# by the worker before the transaction was complete. Needs fixing after the
# upgrade to a Django version > 1.8
"""
now = timezone.now()
if not days:
changed_date = get_last_updated_date()
else:
changed_date = now - timedelta(int(days))
for component in components:
offset = 0
url = URL.format(api_key=settings.REMOZILLA_API_KEY,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timestamp=changed_date,
offset=offset,
limit=LIMIT)
while True:
bugs = requests.get(url).json()
error = bugs.get('error')
# Check the server response for errors
if error:
raise ValueError('Invalid response from server, {0}.'.format(
bugs['message']))
remo_bugs = bugs.get('bugs', [])
if not remo_bugs:
break
for bdata in remo_bugs:
# Get comments for current bug
comment_url = COMMENT_URL.format(
id=bdata['id'], api_key=settings.REMOZILLA_API_KEY)
comments = requests.get(comment_url).json()
error = comments.get('error')
if error:
raise ValueError(
'Invalid response from server, {0}.'.format(
comments['message']))
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
creator_email = bdata['creator']
bug.creator = get_object_or_none(User, email=creator_email)
bug.bug_creation_time = parse_bugzilla_time(
bdata['creation_time'])
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for email in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=email)
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(
User, email=bdata['assigned_to'])
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.bug_last_change_time = parse_bugzilla_time(
bdata.get('last_change_time'))
automated_voting_trigger = 0
bug.budget_needinfo.clear()
bug.council_member_assigned = False
bug.pending_mentor_validation = False
for flag in bdata.get('flags', []):
if flag['status'] == '?' and flag['name'] == BUG_APPROVAL:
automated_voting_trigger += 1
if BUG_WHITEBOARD in bug.whiteboard:
bug.council_member_assigned = True
if ((flag['status'] == '?' and flag['name'] == 'needinfo'
and 'requestee' in flag and flag['requestee']
== (settings.REPS_REVIEW_ALIAS))):
automated_voting_trigger += 1
if flag['status'] == '?' and flag['name'] == BUG_REVIEW:
bug.pending_mentor_validation = True
if (flag['status'] == '?' and flag['name'] == 'needinfo'
and 'requestee' in flag):
email = flag['requestee']
user = get_object_or_none(User, email=email)
if user:
bug.budget_needinfo.add(user)
if automated_voting_trigger == 2 and waffle.switch_is_active(
'automated_polls'):
bug.council_vote_requested = True
unicode_id = str(bdata['id'])
bug_comments = comments['bugs'][unicode_id]['comments']
if bug_comments and bug_comments[0].get('text', ''):
# Enforce unicode encoding.
bug.first_comment = bug_comments[0]['text']
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
login_url = LOGIN_URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD)
response = requests.get(login_url).json()
error = response.get('error')
# Check the server response and get the token
if error:
raise ValueError('Invalid response from server, {0}.'
.format(response['error']))
token = response['token']
now = timezone.now()
if not days:
changed_date = get_last_updated_date()
else:
changed_date = now - timedelta(int(days))
for component in components:
offset = 0
url = URL.format(token=token, component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timestamp=changed_date, offset=offset, limit=LIMIT)
while True:
bugs = requests.get(url).json()
error = bugs.get('error')
# Check the server response and get the token
if error:
raise ValueError('Invalid response from server, {0}.'
.format(bugs['message']))
remo_bugs = bugs.get('bugs', [])
if not remo_bugs:
break
for bdata in remo_bugs:
# Get comments for current bug
comment_url = COMMENT_URL.format(id=bdata['id'], token=token)
comments = requests.get(comment_url).json()
error = comments.get('error')
if error:
raise ValueError('Invalid response from server, {0}.'
.format(comments['message']))
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = unicode(bdata.get('summary', ''))
creator_email = bdata['creator']
bug.creator = get_object_or_none(User, email=creator_email)
bug.bug_creation_time = (
parse_bugzilla_time(bdata['creation_time']))
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for email in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=email)
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = get_object_or_none(
User, email=bdata['assigned_to'])
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.bug_last_change_time = parse_bugzilla_time(
bdata.get('last_change_time'))
automated_voting_trigger = 0
bug.budget_needinfo.clear()
bug.council_member_assigned = False
bug.pending_mentor_validation = False
for flag in bdata.get('flags', []):
if ((flag['status'] == '?' and
flag['name'] == 'remo-approval')):
automated_voting_trigger += 1
if 'Council Reviewer Assigned' in bug.whiteboard:
bug.council_member_assigned = True
if ((flag['status'] == '?' and
flag['name'] == 'needinfo' and 'requestee' in flag and
flag['requestee'] == (settings.REPS_COUNCIL_ALIAS))):
automated_voting_trigger += 1
if flag['status'] == '?' and flag['name'] == 'remo-review':
bug.pending_mentor_validation = True
if (flag['status'] == '?' and flag['name'] == 'needinfo'
and 'requestee' in flag):
email = flag['requestee']
user = get_object_or_none(User, email=email)
if user:
bug.budget_needinfo.add(user)
if ((automated_voting_trigger == 2 and
waffle.switch_is_active('automated_polls'))):
bug.council_vote_requested = True
unicode_id = unicode(bdata['id'])
bug_comments = comments['bugs'][unicode_id]['comments']
if bug_comments and bug_comments[0].get('text', ''):
# Enforce unicode encoding.
bug.first_comment = unicode(bug_comments[0]['text'])
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
def test_with_valid_data(self, fake_get, switch_is_active_mock):
"""Test fetch_bugs valid bug data processing."""
UserFactory.create(username='******')
if ((not getattr(settings, 'REMOZILLA_USERNAME', None)
or not getattr(settings, 'REMOZILLA_PASSWORD', None))):
raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.')
switch_is_active_mock.return_value = True
previous_last_updated_time = get_last_updated_date()
first_request = requests.Request()
first_request.status_code = 200
bug_data = {
'bugs': [{
'id':
7788,
'summary':
'This is summary',
'creator': {
'name': '*****@*****.**'
},
'creation_time':
'2010-10-5T13:45:23Z',
'component':
'Swag Requests',
'whiteboard':
'This is whiteboard',
'cc': [{
'name': '*****@*****.**'
}, {
'name': '*****@*****.**'
}],
'assigned_to': {
'name': '*****@*****.**'
},
'status':
'resolved',
'flags': [{
'status': '?',
'name': 'remo-approval'
}, {
'status': '?',
'name': 'needinfo',
'requestee': {
'name': settings.REPS_COUNCIL_ALIAS
}
}],
'resolution':
'invalid'
}, {
'id':
1199,
'summary':
'New summary',
'creator': {
'name': '*****@*****.**'
},
'creation_time':
'2012-12-5T11:30:23Z',
'component':
'Swag requests',
'cc': [{
'name': '*****@*****.**'
}, {
'name': '*****@*****.**'
}],
'assigned_to': {
'name': '*****@*****.**'
},
'status':
'resolved',
'resolution':
'invalid'
}]
}
first_request.text = json.dumps(bug_data)
second_request = requests.Request()
second_request.status_code = 200
bug_data['bugs'][0]['component'] = 'Planning'
bug_data['bugs'][0]['id'] = 7789
bug_data['bugs'][1]['component'] = 'Planning'
bug_data['bugs'][1]['id'] = 1200
second_request.text = json.dumps(bug_data)
empty_request = requests.Request()
empty_request.status_code = 200
empty_request.text = json.dumps({'bugs': []})
values = [first_request, empty_request, second_request, empty_request]
fake_get.side_effect = values
fetch_bugs(components=['Planning', 'Swag Requests'])
eq_(Bug.objects.all().count(), 4)
eq_(Bug.objects.filter(component='Planning').count(), 2)
eq_(Bug.objects.filter(component='Swag Requests').count(), 2)
# refresh status_obj
self.assertGreater(get_last_updated_date(), previous_last_updated_time)
bug = Bug.objects.get(bug_id=7788)
eq_(bug.cc.all().count(), 1)
eq_(bug.assigned_to.email, '*****@*****.**')
eq_(bug.resolution, 'INVALID')
ok_(bug.council_vote_requested)
bug = Bug.objects.get(bug_id=1199)
eq_(bug.creator, None)
eq_(bug.cc.all().count(), 1)
ok_(not bug.council_vote_requested)
def fetch_bugs(components=COMPONENTS, days=None):
"""Fetch all bugs from Bugzilla.
Loop over components and fetch bugs updated the last days. Link
Bugzilla users with users on this website, when possible.
"""
now = timezone.now()
if not days:
days = (now - get_last_updated_date()).days + 1
for component in components:
offset = 0
url = URL.format(username=settings.REMOZILLA_USERNAME,
password=settings.REMOZILLA_PASSWORD,
component=quote(component),
fields=','.join(BUGZILLA_FIELDS),
timedelta=days,
offset=offset,
limit=LIMIT)
while True:
response = requests.get(url)
if response.status_code != 200:
raise ValueError('Invalid response from server.')
bugs = json.loads(response.text)
if not bugs['bugs']:
break
for bdata in bugs['bugs']:
bug, created = Bug.objects.get_or_create(bug_id=bdata['id'])
bug.summary = bdata.get('summary', '')
creator_name = bdata['creator']['name']
bug.creator = get_object_or_none(User, email=creator_name)
creation_time = datetime.strptime(bdata['creation_time'],
'%Y-%m-%dT%H:%M:%SZ')
bug.bug_creation_time = creation_time
bug.component = bdata['component']
bug.whiteboard = bdata.get('whiteboard', '')
bug.cc.clear()
for person in bdata.get('cc', []):
cc_user = get_object_or_none(User, email=person['name'])
if cc_user:
bug.cc.add(cc_user)
bug.assigned_to = (get_object_or_none(
User, email=(bdata['assigned_to']['name'])))
bug.status = bdata['status']
bug.resolution = bdata.get('resolution', '')
bug.due_date = bdata.get('cf_due_date', None)
if 'last_change_time' in bdata:
bug.bug_last_change_time = datetime.strptime(
bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ')
flags = bdata.get('flags', [])
bug.flag_status = next(
(item['status']
for item in flags if item['status'] == '?'), '')
bug.flag_name = next(
(item['name']
for item in flags if item['name'] == 'remo-review'), '')
comments = bdata.get('comments', [])
if comments and comments[0].get('text', ''):
bug.first_comment = comments[0]['text']
bug.save()
offset += LIMIT
url = urlparams(url, offset=offset)
set_last_updated_date(now)
