def test_with_valid_data(self, fake_requests_obj): """Test fetch_bugs valid bug data processing.""" if (not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None)): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') previous_last_updated_time = get_last_updated_date() request = requests.Request() request.status_code = 200 bug_data = {'bugs': [{'id': 7788, 'summary':'This is summary', 'creator': {'name': '*****@*****.**'}, 'creation_time': '2010-10-5T13:45:23Z', 'component':'Swag Requests', 'whiteboard':'This is whiteboard', 'cc': [{'name': '*****@*****.**'}, {'name': '*****@*****.**'}], 'assigned_to':{'name':'*****@*****.**'}, 'status':'resolved', 'resolution':'invalid', 'cf_due_date': None}, {'id': 1199, 'summary':'New summary', 'creator': {'name': '*****@*****.**'}, 'creation_time': '2012-12-5T11:30:23Z', 'component':'Budget Requests', 'cc': [{'name': '*****@*****.**'}, {'name': '*****@*****.**'}], 'assigned_to':{'name':'*****@*****.**'}, 'status':'resolved', 'resolution':'invalid', 'cf_due_date': None}]} request.text = json.dumps(bug_data) (fake_requests_obj.expects_call().returns(request)) fetch_bugs() eq_(Bug.objects.all().count(), len(bug_data['bugs'])) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.due_date, None) eq_(bug.resolution, 'INVALID') bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ now = timezone.now() if not days: days = (now - get_last_updated_date()).days + 1 for component in COMPONENTS: url = URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timedelta=days) response = requests.get(url) if response.status_code != 200: raise ValueError('Invalid response from server.') bugs = json.loads(response.text) for bdata in bugs['bugs']: bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = bdata.get('summary', '') bug.creator = get_object_or_none(User, email=bdata['creator']['name']) bug.bug_creation_time = datetime.strptime(bdata['creation_time'], '%Y-%m-%dT%H:%M:%SZ') bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for person in bdata.get('cc', []): cc_user = get_object_or_none(User, email=person['name']) if cc_user: bug.cc.add(cc_user) bug.assigned_to = get_object_or_none(User, email=(bdata['assigned_to']\ ['name'])) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.due_date = bdata.get('cf_due_date', None) if 'last_change_time' in bdata: bug.bug_last_change_time = datetime.strptime( bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ') bug.save() set_last_updated_date(now)
def test_with_valid_data(self, fake_get, switch_is_active_mock): """Test fetch_bugs valid bug data processing.""" UserFactory.create(username='******') if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None))): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') switch_is_active_mock.return_value = True previous_last_updated_time = get_last_updated_date() first_request = requests.Request() first_request.status_code = 200 user = UserFactory.create(groups=['Rep'], email='*****@*****.**') bug_data = {'bugs': [{'id': 7788, 'summary': 'This is summary', 'creator': {'name': '*****@*****.**'}, 'creation_time': '2010-10-5T13:45:23Z', 'component': 'Budget Requests', 'whiteboard': 'This is whiteboard', 'cc': [{'name': '*****@*****.**'}, {'name': '*****@*****.**'}], 'assigned_to': {'name': '*****@*****.**'}, 'status': 'resolved', 'flags': [ {'status': '?', 'name': 'remo-approval'}, {'status': '?', 'name': 'needinfo', 'requestee': { 'name': settings.REPS_COUNCIL_ALIAS}}, {'status': '?', 'name': 'needinfo', 'requestee': { 'name': '*****@*****.**'}} ], 'resolution': 'invalid'}, {'id': 1199, 'summary': 'New summary', 'creator': {'name': '*****@*****.**'}, 'creation_time': '2012-12-5T11:30:23Z', 'component': 'Budget Requests', 'whiteboard': 'Council Reviewer Assigned', 'cc': [{'name': '*****@*****.**'}, {'name': '*****@*****.**'}], 'flags': [{'status': '?', 'name': 'remo-approval'}, {'status': '?', 'name': 'remo-review'}], 'assigned_to': {'name': '*****@*****.**'}, 'status': 'resolved', 'resolution': 'invalid'}]} first_request.text = json.dumps(bug_data) second_request = requests.Request() second_request.status_code = 200 bug_data['bugs'][0]['component'] = 'Planning' bug_data['bugs'][0]['id'] = 7789 bug_data['bugs'][1]['component'] = 'Planning' bug_data['bugs'][1]['id'] = 1200 second_request.text = json.dumps(bug_data) empty_request = requests.Request() empty_request.status_code = 200 empty_request.text = json.dumps({'bugs': []}) values = [first_request, empty_request, second_request, empty_request] fake_get.side_effect = values fetch_bugs(components=['Planning', 'Budget Requests']) eq_(Bug.objects.all().count(), 4) eq_(Bug.objects.filter(component='Planning').count(), 2) eq_(Bug.objects.filter(component='Budget Requests').count(), 2) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.resolution, 'INVALID') ok_(bug.council_vote_requested) ok_(user in bug.budget_needinfo.all()) bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1) ok_(not bug.council_vote_requested) ok_(bug.pending_mentor_validation) ok_(bug.council_member_assigned)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ login_url = LOGIN_URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD) response = requests.get(login_url).json() error = response.get('error') # Check the server response and get the token if error: raise ValueError('Invalid response from server, {0}.'.format( response['error'])) token = response['token'] now = timezone.now() if not days: changed_date = get_last_updated_date() else: changed_date = now - timedelta(int(days)) for component in components: offset = 0 url = URL.format(token=token, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timestamp=changed_date, offset=offset, limit=LIMIT) while True: bugs = requests.get(url).json() error = bugs.get('error') # Check the server response and get the token if error: raise ValueError('Invalid response from server, {0}.'.format( bugs['message'])) remo_bugs = bugs.get('bugs', []) if not remo_bugs: break for bdata in remo_bugs: # Get comments for current bug comment_url = COMMENT_URL.format(id=bdata['id'], token=token) comments = requests.get(comment_url).json() error = comments.get('error') if error: raise ValueError( 'Invalid response from server, {0}.'.format( comments['message'])) bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = unicode(bdata.get('summary', '')) creator_email = bdata['creator'] bug.creator = get_object_or_none(User, email=creator_email) bug.bug_creation_time = (parse_bugzilla_time( bdata['creation_time'])) bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for email in bdata.get('cc', []): cc_user = get_object_or_none(User, email=email) if cc_user: bug.cc.add(cc_user) bug.assigned_to = get_object_or_none( User, email=bdata['assigned_to']) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.bug_last_change_time = parse_bugzilla_time( bdata.get('last_change_time')) automated_voting_trigger = 0 bug.budget_needinfo.clear() bug.council_member_assigned = False bug.pending_mentor_validation = False for flag in bdata.get('flags', []): if ((flag['status'] == '?' and flag['name'] == 'remo-approval')): automated_voting_trigger += 1 if 'Council Reviewer Assigned' in bug.whiteboard: bug.council_member_assigned = True if ((flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag and flag['requestee'] == (settings.REPS_COUNCIL_ALIAS))): automated_voting_trigger += 1 if flag['status'] == '?' and flag['name'] == 'remo-review': bug.pending_mentor_validation = True if (flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag): email = flag['requestee'] user = get_object_or_none(User, email=email) if user: bug.budget_needinfo.add(user) if ((automated_voting_trigger == 2 and waffle.switch_is_active('automated_polls'))): bug.council_vote_requested = True unicode_id = unicode(bdata['id']) bug_comments = comments['bugs'][unicode_id]['comments'] if bug_comments and bug_comments[0].get('text', ''): # Enforce unicode encoding. bug.first_comment = unicode(bug_comments[0]['text']) bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)
def test_with_valid_data(self, mocked_request, switch_is_active_mock): """Test fetch_bugs valid bug data processing.""" UserFactory.create(username='******') if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None))): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') switch_is_active_mock.return_value = True previous_last_updated_time = get_last_updated_date() mentor = UserFactory.create() user = UserFactory.create(groups=['Rep'], email='*****@*****.**', userprofile__mentor=mentor) login_data = {u'token': u'bugzilla_token', u'id': 12345} bug_data = [{'id': 7788, 'summary': 'This is summary', 'creator': '*****@*****.**', 'creation_time': '2010-10-5T13:45:23Z', 'component': 'Budget Requests', 'whiteboard': 'This is whiteboard', 'cc': ['*****@*****.**', '*****@*****.**'], 'assigned_to': '*****@*****.**', 'status': 'resolved', 'flags': [{'status': '?', 'name': 'remo-approval'}, {'status': '?', 'name': 'needinfo', 'requestee': settings.REPS_REVIEW_ALIAS}, {'status': '?', 'name': 'needinfo', 'requestee': '*****@*****.**'}], 'resolution': 'invalid'}, {'id': 1199, 'summary': 'New summary', 'creator': '*****@*****.**', 'creation_time': '2012-12-5T11:30:23Z', 'component': 'Budget Requests', 'whiteboard': 'Review Team approval needed', 'cc': ['*****@*****.**', '*****@*****.**'], 'flags': [{'status': '?', 'name': 'remo-approval'}, {'status': '?', 'name': 'remo-review'}], 'assigned_to': '*****@*****.**', 'status': 'resolved', 'resolution': 'invalid'}] def mocked_get(url, *args, **kwargs): mocked_response = Mock() if 'login' in url: mocked_response.json.return_value = login_data return mocked_response elif 'comment' in url: comments = { 'bugs': { '7788': { 'comments': [{'text': 'foo'}, {'text': 'bar'}] }, '1199': { 'comments': [{'text': 'bar'}, {'text': 'foo'}] } } } mocked_response.json.return_value = comments return mocked_response else: mocked_response.json.return_value = {'bugs': bug_data} url_params = parse_qs(urlparse(url).query) offset = url_params.get('offset') if offset and int(offset[0]) > 0: mocked_response.json.return_value = {'bugs': []} return mocked_response mocked_request.side_effect = mocked_get fetch_bugs() eq_(Bug.objects.all().count(), 2) eq_(Bug.objects.filter(component='Budget Requests').count(), 2) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.resolution, 'INVALID') eq_(bug.creator, User.objects.get(email='*****@*****.**')) ok_(bug.council_vote_requested) ok_(user in bug.budget_needinfo.all()) eq_(bug.first_comment, 'foo') bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1) ok_(not bug.council_vote_requested) ok_(bug.pending_mentor_validation) ok_(bug.council_member_assigned) eq_(bug.first_comment, 'bar')
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ now = timezone.now() if not days: days = (now - get_last_updated_date()).days + 1 for component in components: offset = 0 url = URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timedelta=days, offset=offset, limit=LIMIT) while True: response = requests.get(url) if response.status_code != 200: raise ValueError('Invalid response from server.') bugs = json.loads(response.text) if not bugs['bugs']: break for bdata in bugs['bugs']: bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = bdata.get('summary', '') creator_name = bdata['creator']['name'] bug.creator = get_object_or_none(User, email=creator_name) creation_time = datetime.strptime(bdata['creation_time'], '%Y-%m-%dT%H:%M:%SZ') bug.bug_creation_time = creation_time bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for person in bdata.get('cc', []): cc_user = get_object_or_none(User, email=person['name']) if cc_user: bug.cc.add(cc_user) bug.assigned_to = ( get_object_or_none(User, email=(bdata['assigned_to']['name']))) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.due_date = bdata.get('cf_due_date', None) if 'last_change_time' in bdata: bug.bug_last_change_time = datetime.strptime( bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ') flags = bdata.get('flags', []) bug.flag_status = next((item['status'] for item in flags if item['status'] == '?'), '') bug.flag_name = next((item['name'] for item in flags if item['name'] == 'remo-review'), '') comments = bdata.get('comments', []) if comments and comments[0].get('text', ''): bug.first_comment = comments[0]['text'] bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)
def test_with_valid_data(self, mocked_request, switch_is_active_mock): """Test fetch_bugs valid bug data processing.""" UserFactory.create(username='******') if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None))): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') switch_is_active_mock.return_value = True previous_last_updated_time = get_last_updated_date() mentor = UserFactory.create() user = UserFactory.create(groups=['Rep'], email='*****@*****.**', userprofile__mentor=mentor) login_data = {u'token': u'bugzilla_token', u'id': 12345} bug_data = [{'id': 7788, 'summary': 'This is summary', 'creator': '*****@*****.**', 'creation_time': '2010-10-5T13:45:23Z', 'component': 'Budget Requests', 'whiteboard': 'This is whiteboard', 'cc': ['*****@*****.**', '*****@*****.**'], 'assigned_to': '*****@*****.**', 'status': 'resolved', 'flags': [{'status': '?', 'name': 'remo-approval'}, {'status': '?', 'name': 'needinfo', 'requestee': settings.REPS_COUNCIL_ALIAS}, {'status': '?', 'name': 'needinfo', 'requestee': '*****@*****.**'}], 'resolution': 'invalid'}, {'id': 1199, 'summary': 'New summary', 'creator': '*****@*****.**', 'creation_time': '2012-12-5T11:30:23Z', 'component': 'Budget Requests', 'whiteboard': 'Council Reviewer Assigned', 'cc': ['*****@*****.**', '*****@*****.**'], 'flags': [{'status': '?', 'name': 'remo-approval'}, {'status': '?', 'name': 'remo-review'}], 'assigned_to': '*****@*****.**', 'status': 'resolved', 'resolution': 'invalid'}] def mocked_get(url, *args, **kwargs): mocked_response = Mock() if 'login' in url: mocked_response.json.return_value = login_data return mocked_response elif 'comment' in url: comments = { 'bugs': { '7788': { 'comments': [{'text': 'foo'}, {'text': 'bar'}] }, '1199': { 'comments': [{'text': 'bar'}, {'text': 'foo'}] } } } mocked_response.json.return_value = comments return mocked_response else: mocked_response.json.return_value = {'bugs': bug_data} url_params = parse_qs(urlparse(url).query) offset = url_params.get('offset') if offset and int(offset[0]) > 0: mocked_response.json.return_value = {'bugs': []} return mocked_response mocked_request.side_effect = mocked_get fetch_bugs() eq_(Bug.objects.all().count(), 2) eq_(Bug.objects.filter(component='Budget Requests').count(), 2) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.resolution, 'INVALID') eq_(bug.creator, User.objects.get(email='*****@*****.**')) ok_(bug.council_vote_requested) ok_(user in bug.budget_needinfo.all()) eq_(bug.first_comment, 'foo') bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1) ok_(not bug.council_vote_requested) ok_(bug.pending_mentor_validation) ok_(bug.council_member_assigned) eq_(bug.first_comment, 'bar')
def test_with_valid_data(self, fake_requests_obj): """Test fetch_bugs valid bug data processing.""" if (not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None)): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') previous_last_updated_time = get_last_updated_date() first_request = requests.Request() first_request.status_code = 200 bug_data = { 'bugs': [{ 'id': 7788, 'summary': 'This is summary', 'creator': { 'name': '*****@*****.**' }, 'creation_time': '2010-10-5T13:45:23Z', 'component': 'Swag Requests', 'whiteboard': 'This is whiteboard', 'cc': [{ 'name': '*****@*****.**' }, { 'name': '*****@*****.**' }], 'assigned_to': { 'name': '*****@*****.**' }, 'status': 'resolved', 'resolution': 'invalid', 'cf_due_date': None }, { 'id': 1199, 'summary': 'New summary', 'creator': { 'name': '*****@*****.**' }, 'creation_time': '2012-12-5T11:30:23Z', 'component': 'Swag requests', 'cc': [{ 'name': '*****@*****.**' }, { 'name': '*****@*****.**' }], 'assigned_to': { 'name': '*****@*****.**' }, 'status': 'resolved', 'resolution': 'invalid', 'cf_due_date': None }] } first_request.text = json.dumps(bug_data) second_request = requests.Request() second_request.status_code = 200 bug_data['bugs'][0]['component'] = 'Planning' bug_data['bugs'][0]['id'] = 7789 bug_data['bugs'][1]['component'] = 'Planning' bug_data['bugs'][1]['id'] = 1200 second_request.text = json.dumps(bug_data) empty_request = requests.Request() empty_request.status_code = 200 empty_request.text = json.dumps({'bugs': []}) fake_requests_obj.expects_call() fake_requests_obj.returns(first_request) fake_requests_obj.next_call().returns(empty_request) fake_requests_obj.next_call().returns(second_request) fake_requests_obj.next_call().returns(empty_request) fetch_bugs(components=['Planning', 'Swag Requests']) eq_(Bug.objects.all().count(), 4) eq_(Bug.objects.filter(component='Planning').count(), 2) eq_(Bug.objects.filter(component='Swag Requests').count(), 2) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.due_date, None) eq_(bug.resolution, 'INVALID') bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ now = timezone.now() if not days: days = (now - get_last_updated_date()).days + 1 for component in components: offset = 0 url = URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timedelta=days, offset=offset, limit=LIMIT) while True: response = requests.get(url) if response.status_code != 200: raise ValueError('Invalid response from server.') bugs = json.loads(response.text) if not bugs['bugs']: break for bdata in bugs['bugs']: bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = bdata.get('summary', '') creator_name = bdata['creator']['name'] bug.creator = get_object_or_none(User, email=creator_name) bug.bug_creation_time = ( parse_bugzilla_time(bdata['creation_time'])) bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for person in bdata.get('cc', []): cc_user = get_object_or_none(User, email=person['name']) if cc_user: bug.cc.add(cc_user) bug.assigned_to = get_object_or_none( User, email=bdata['assigned_to']['name']) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.bug_last_change_time = parse_bugzilla_time( bdata.get('last_change_time')) automated_voting_trigger = 0 for flag in bdata.get('flags', []): if ((flag['status'] == '?' and flag['name'] == 'remo-approval')): automated_voting_trigger += 1 if ((flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag and flag['requestee']['name'] == ( settings.REPS_COUNCIL_ALIAS))): automated_voting_trigger += 1 if ((automated_voting_trigger == 2 and waffle.switch_is_active('automated_polls'))): bug.council_vote_requested = True comments = bdata.get('comments', []) if comments and comments[0].get('text', ''): bug.first_comment = comments[0]['text'] bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)
def test_with_valid_data(self, fake_requests_obj): """Test fetch_bugs valid bug data processing.""" if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None))): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') previous_last_updated_time = get_last_updated_date() first_request = requests.Request() first_request.status_code = 200 bug_data = {'bugs': [{'id': 7788, 'summary': 'This is summary', 'creator': {'name': '*****@*****.**'}, 'creation_time': '2010-10-5T13:45:23Z', 'component': 'Swag Requests', 'whiteboard': 'This is whiteboard', 'cc': [{'name': '*****@*****.**'}, {'name': '*****@*****.**'}], 'assigned_to': {'name': '*****@*****.**'}, 'status': 'resolved', 'flags': [ {'requestee': { 'ref': '', 'name': '*****@*****.**'}, 'status': '?', 'name': 'remo-review'}], 'resolution': 'invalid'}, {'id': 1199, 'summary': 'New summary', 'creator': {'name': '*****@*****.**'}, 'creation_time': '2012-12-5T11:30:23Z', 'component': 'Swag requests', 'cc': [{'name': '*****@*****.**'}, {'name': '*****@*****.**'}], 'assigned_to': {'name': '*****@*****.**'}, 'status': 'resolved', 'resolution': 'invalid'}]} first_request.text = json.dumps(bug_data) second_request = requests.Request() second_request.status_code = 200 bug_data['bugs'][0]['component'] = 'Planning' bug_data['bugs'][0]['id'] = 7789 bug_data['bugs'][1]['component'] = 'Planning' bug_data['bugs'][1]['id'] = 1200 second_request.text = json.dumps(bug_data) empty_request = requests.Request() empty_request.status_code = 200 empty_request.text = json.dumps({'bugs': []}) fake_requests_obj.expects_call() fake_requests_obj.returns(first_request) fake_requests_obj.next_call().returns(empty_request) fake_requests_obj.next_call().returns(second_request) fake_requests_obj.next_call().returns(empty_request) fetch_bugs(components=['Planning', 'Swag Requests']) eq_(Bug.objects.all().count(), 4) eq_(Bug.objects.filter(component='Planning').count(), 2) eq_(Bug.objects.filter(component='Swag Requests').count(), 2) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.resolution, 'INVALID') eq_(bug.council_vote_requested, True) bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1) eq_(bug.council_vote_requested, False)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ now = timezone.now() if not days: days = (now - get_last_updated_date()).days + 1 for component in components: offset = 0 url = URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timedelta=days, offset=offset, limit=LIMIT) while True: response = requests.get(url) if response.status_code != 200: raise ValueError('Invalid response from server.') bugs = json.loads(response.text) if not bugs['bugs']: break for bdata in bugs['bugs']: bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = bdata.get('summary', '') creator_name = bdata['creator']['name'] bug.creator = get_object_or_none(User, email=creator_name) bug.bug_creation_time = ( parse_bugzilla_time(bdata['creation_time'])) bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for person in bdata.get('cc', []): cc_user = get_object_or_none(User, email=person['name']) if cc_user: bug.cc.add(cc_user) bug.assigned_to = get_object_or_none( User, email=bdata['assigned_to']['name']) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.bug_last_change_time = parse_bugzilla_time( bdata.get('last_change_time')) automated_voting_trigger = 0 bug.budget_needinfo.clear() bug.council_member_assigned = False bug.pending_mentor_validation = False for flag in bdata.get('flags', []): if ((flag['status'] == '?' and flag['name'] == 'remo-approval')): automated_voting_trigger += 1 if 'Council Reviewer Assigned' in bug.whiteboard: bug.council_member_assigned = True if ((flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag and flag['requestee']['name'] == ( settings.REPS_COUNCIL_ALIAS))): automated_voting_trigger += 1 if ((flag['status'] == '?' and flag['name'] == 'remo-review')): bug.pending_mentor_validation = True if ((flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag and bug.component == 'Budget Requests')): email = flag['requestee']['name'] user = get_object_or_none(User, email=email) if user: bug.budget_needinfo.add(user) if ((automated_voting_trigger == 2 and waffle.switch_is_active('automated_polls'))): bug.council_vote_requested = True comments = bdata.get('comments', []) if comments and comments[0].get('text', ''): bug.first_comment = comments[0]['text'] bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. # TODO: This can trigger a does not exist error because the task was picked # by the worker before the transaction was complete. Needs fixing after the # upgrade to a Django version > 1.8 """ now = timezone.now() if not days: changed_date = get_last_updated_date() else: changed_date = now - timedelta(int(days)) for component in components: offset = 0 url = URL.format(api_key=settings.REMOZILLA_API_KEY, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timestamp=changed_date, offset=offset, limit=LIMIT) while True: bugs = requests.get(url).json() error = bugs.get('error') # Check the server response for errors if error: raise ValueError('Invalid response from server, {0}.'.format( bugs['message'])) remo_bugs = bugs.get('bugs', []) if not remo_bugs: break for bdata in remo_bugs: # Get comments for current bug comment_url = COMMENT_URL.format( id=bdata['id'], api_key=settings.REMOZILLA_API_KEY) comments = requests.get(comment_url).json() error = comments.get('error') if error: raise ValueError( 'Invalid response from server, {0}.'.format( comments['message'])) bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = bdata.get('summary', '') creator_email = bdata['creator'] bug.creator = get_object_or_none(User, email=creator_email) bug.bug_creation_time = parse_bugzilla_time( bdata['creation_time']) bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for email in bdata.get('cc', []): cc_user = get_object_or_none(User, email=email) if cc_user: bug.cc.add(cc_user) bug.assigned_to = get_object_or_none( User, email=bdata['assigned_to']) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.bug_last_change_time = parse_bugzilla_time( bdata.get('last_change_time')) automated_voting_trigger = 0 bug.budget_needinfo.clear() bug.council_member_assigned = False bug.pending_mentor_validation = False for flag in bdata.get('flags', []): if flag['status'] == '?' and flag['name'] == BUG_APPROVAL: automated_voting_trigger += 1 if BUG_WHITEBOARD in bug.whiteboard: bug.council_member_assigned = True if ((flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag and flag['requestee'] == (settings.REPS_REVIEW_ALIAS))): automated_voting_trigger += 1 if flag['status'] == '?' and flag['name'] == BUG_REVIEW: bug.pending_mentor_validation = True if (flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag): email = flag['requestee'] user = get_object_or_none(User, email=email) if user: bug.budget_needinfo.add(user) if automated_voting_trigger == 2 and waffle.switch_is_active( 'automated_polls'): bug.council_vote_requested = True unicode_id = str(bdata['id']) bug_comments = comments['bugs'][unicode_id]['comments'] if bug_comments and bug_comments[0].get('text', ''): # Enforce unicode encoding. bug.first_comment = bug_comments[0]['text'] bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ login_url = LOGIN_URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD) response = requests.get(login_url).json() error = response.get('error') # Check the server response and get the token if error: raise ValueError('Invalid response from server, {0}.' .format(response['error'])) token = response['token'] now = timezone.now() if not days: changed_date = get_last_updated_date() else: changed_date = now - timedelta(int(days)) for component in components: offset = 0 url = URL.format(token=token, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timestamp=changed_date, offset=offset, limit=LIMIT) while True: bugs = requests.get(url).json() error = bugs.get('error') # Check the server response and get the token if error: raise ValueError('Invalid response from server, {0}.' .format(bugs['message'])) remo_bugs = bugs.get('bugs', []) if not remo_bugs: break for bdata in remo_bugs: # Get comments for current bug comment_url = COMMENT_URL.format(id=bdata['id'], token=token) comments = requests.get(comment_url).json() error = comments.get('error') if error: raise ValueError('Invalid response from server, {0}.' .format(comments['message'])) bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = unicode(bdata.get('summary', '')) creator_email = bdata['creator'] bug.creator = get_object_or_none(User, email=creator_email) bug.bug_creation_time = ( parse_bugzilla_time(bdata['creation_time'])) bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for email in bdata.get('cc', []): cc_user = get_object_or_none(User, email=email) if cc_user: bug.cc.add(cc_user) bug.assigned_to = get_object_or_none( User, email=bdata['assigned_to']) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.bug_last_change_time = parse_bugzilla_time( bdata.get('last_change_time')) automated_voting_trigger = 0 bug.budget_needinfo.clear() bug.council_member_assigned = False bug.pending_mentor_validation = False for flag in bdata.get('flags', []): if ((flag['status'] == '?' and flag['name'] == 'remo-approval')): automated_voting_trigger += 1 if 'Council Reviewer Assigned' in bug.whiteboard: bug.council_member_assigned = True if ((flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag and flag['requestee'] == (settings.REPS_COUNCIL_ALIAS))): automated_voting_trigger += 1 if flag['status'] == '?' and flag['name'] == 'remo-review': bug.pending_mentor_validation = True if (flag['status'] == '?' and flag['name'] == 'needinfo' and 'requestee' in flag): email = flag['requestee'] user = get_object_or_none(User, email=email) if user: bug.budget_needinfo.add(user) if ((automated_voting_trigger == 2 and waffle.switch_is_active('automated_polls'))): bug.council_vote_requested = True unicode_id = unicode(bdata['id']) bug_comments = comments['bugs'][unicode_id]['comments'] if bug_comments and bug_comments[0].get('text', ''): # Enforce unicode encoding. bug.first_comment = unicode(bug_comments[0]['text']) bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)
def test_with_valid_data(self, fake_get, switch_is_active_mock): """Test fetch_bugs valid bug data processing.""" UserFactory.create(username='******') if ((not getattr(settings, 'REMOZILLA_USERNAME', None) or not getattr(settings, 'REMOZILLA_PASSWORD', None))): raise SkipTest('Skipping test due to unset REMOZILLA_USERNAME.') switch_is_active_mock.return_value = True previous_last_updated_time = get_last_updated_date() first_request = requests.Request() first_request.status_code = 200 bug_data = { 'bugs': [{ 'id': 7788, 'summary': 'This is summary', 'creator': { 'name': '*****@*****.**' }, 'creation_time': '2010-10-5T13:45:23Z', 'component': 'Swag Requests', 'whiteboard': 'This is whiteboard', 'cc': [{ 'name': '*****@*****.**' }, { 'name': '*****@*****.**' }], 'assigned_to': { 'name': '*****@*****.**' }, 'status': 'resolved', 'flags': [{ 'status': '?', 'name': 'remo-approval' }, { 'status': '?', 'name': 'needinfo', 'requestee': { 'name': settings.REPS_COUNCIL_ALIAS } }], 'resolution': 'invalid' }, { 'id': 1199, 'summary': 'New summary', 'creator': { 'name': '*****@*****.**' }, 'creation_time': '2012-12-5T11:30:23Z', 'component': 'Swag requests', 'cc': [{ 'name': '*****@*****.**' }, { 'name': '*****@*****.**' }], 'assigned_to': { 'name': '*****@*****.**' }, 'status': 'resolved', 'resolution': 'invalid' }] } first_request.text = json.dumps(bug_data) second_request = requests.Request() second_request.status_code = 200 bug_data['bugs'][0]['component'] = 'Planning' bug_data['bugs'][0]['id'] = 7789 bug_data['bugs'][1]['component'] = 'Planning' bug_data['bugs'][1]['id'] = 1200 second_request.text = json.dumps(bug_data) empty_request = requests.Request() empty_request.status_code = 200 empty_request.text = json.dumps({'bugs': []}) values = [first_request, empty_request, second_request, empty_request] fake_get.side_effect = values fetch_bugs(components=['Planning', 'Swag Requests']) eq_(Bug.objects.all().count(), 4) eq_(Bug.objects.filter(component='Planning').count(), 2) eq_(Bug.objects.filter(component='Swag Requests').count(), 2) # refresh status_obj self.assertGreater(get_last_updated_date(), previous_last_updated_time) bug = Bug.objects.get(bug_id=7788) eq_(bug.cc.all().count(), 1) eq_(bug.assigned_to.email, '*****@*****.**') eq_(bug.resolution, 'INVALID') ok_(bug.council_vote_requested) bug = Bug.objects.get(bug_id=1199) eq_(bug.creator, None) eq_(bug.cc.all().count(), 1) ok_(not bug.council_vote_requested)
def fetch_bugs(components=COMPONENTS, days=None): """Fetch all bugs from Bugzilla. Loop over components and fetch bugs updated the last days. Link Bugzilla users with users on this website, when possible. """ now = timezone.now() if not days: days = (now - get_last_updated_date()).days + 1 for component in components: offset = 0 url = URL.format(username=settings.REMOZILLA_USERNAME, password=settings.REMOZILLA_PASSWORD, component=quote(component), fields=','.join(BUGZILLA_FIELDS), timedelta=days, offset=offset, limit=LIMIT) while True: response = requests.get(url) if response.status_code != 200: raise ValueError('Invalid response from server.') bugs = json.loads(response.text) if not bugs['bugs']: break for bdata in bugs['bugs']: bug, created = Bug.objects.get_or_create(bug_id=bdata['id']) bug.summary = bdata.get('summary', '') creator_name = bdata['creator']['name'] bug.creator = get_object_or_none(User, email=creator_name) creation_time = datetime.strptime(bdata['creation_time'], '%Y-%m-%dT%H:%M:%SZ') bug.bug_creation_time = creation_time bug.component = bdata['component'] bug.whiteboard = bdata.get('whiteboard', '') bug.cc.clear() for person in bdata.get('cc', []): cc_user = get_object_or_none(User, email=person['name']) if cc_user: bug.cc.add(cc_user) bug.assigned_to = (get_object_or_none( User, email=(bdata['assigned_to']['name']))) bug.status = bdata['status'] bug.resolution = bdata.get('resolution', '') bug.due_date = bdata.get('cf_due_date', None) if 'last_change_time' in bdata: bug.bug_last_change_time = datetime.strptime( bdata['last_change_time'], '%Y-%m-%dT%H:%M:%SZ') flags = bdata.get('flags', []) bug.flag_status = next( (item['status'] for item in flags if item['status'] == '?'), '') bug.flag_name = next( (item['name'] for item in flags if item['name'] == 'remo-review'), '') comments = bdata.get('comments', []) if comments and comments[0].get('text', ''): bug.first_comment = comments[0]['text'] bug.save() offset += LIMIT url = urlparams(url, offset=offset) set_last_updated_date(now)