def update_user(self, record): email = record.get('author_email') user_e = utils.load_user(self.runtime_storage_inst, email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if email and (not user_e) and (not launchpad_id): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: user_name = lp_user_name user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {} user = self._create_user(launchpad_id, email, user_name) if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'): # sequence numbers are set and the same, merge is not needed user = user_e else: if user_e or user_l: user = self._merge_user_profiles(user_e, user_l, user) else: # create new if not user_name: user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) utils.store_user(self.runtime_storage_inst, user) return user
def update_user(self, record): email = record.get('author_email') user = utils.load_user(self.runtime_storage_inst, email) if not user: if record.get('launchpad_id'): launchpad_id = record.get('launchpad_id') user_name = record.get('author_name') else: launchpad_id, user_name = self._get_lp_info(email) if launchpad_id: user = utils.load_user(self.runtime_storage_inst, launchpad_id) if user: # merge emails if email: self._update_user_profile(user, email) else: # create new if not user_name: user_name = record.get('author_name') if not user_name: user_name = self._get_lp_user_name(launchpad_id) user = self._create_user(launchpad_id, email, user_name) utils.store_user(self.runtime_storage_inst, user) return user
def update_user(self, record): email = record.get('author_email') user_e = utils.load_user(self.runtime_storage_inst, email) or {} user_name = record.get('author_name') launchpad_id = record.get('launchpad_id') if email and (not user_e) and (not launchpad_id): # query LP launchpad_id, lp_user_name = self._get_lp_info(email) if lp_user_name: user_name = lp_user_name user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {} user = self._create_user(launchpad_id, email, user_name) if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'): # sequence numbers are set and the same, merge is not needed user = user_e else: if user_e or user_l: user = self._merge_user_profiles([user_e, user_l, user]) else: # create new if not user_name: user_name = self._get_lp_user_name(launchpad_id) if user_name: user['user_name'] = user_name LOG.debug('Created new user: %s', user) utils.store_user(self.runtime_storage_inst, user) return user
def test_process_email_then_review(self): # it is expected that the user profile will contain both email and # LP id record_processor_inst = self.make_record_processor() list(record_processor_inst.process([ {'record_type': 'email', 'message_id': '<message-id>', 'author_email': '*****@*****.**', 'subject': 'hello, world!', 'body': 'lorem ipsum', 'date': 1234567890}, {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'John Doe', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1379404951, 'module': 'nova', 'branch': 'master'} ])) user = {'seq': 1, 'core': [], 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, '*****@*****.**')) self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe'))
def test_merge_users(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'} }, companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], ) runtime_storage_inst = record_processor_inst.runtime_storage_inst runtime_storage_inst.set_records(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890}, {'record_type': 'email', 'message_id': '<message-id>', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'subject': 'hello, world!', 'body': 'lorem ipsum', 'date': 1234567890}, {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'John Doe', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1379404951, 'module': 'nova', 'branch': 'master'} ])) record_processor_inst.update() user = {'seq': 2, 'core': [], 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{'company_name': 'IBM', 'end_date': 0}]} runtime_storage_inst = record_processor_inst.runtime_storage_inst self.assertEqual(2, runtime_storage_inst.get_by_key('user:count')) self.assertEqual(None, utils.load_user(runtime_storage_inst, 1)) self.assertEqual(user, utils.load_user(runtime_storage_inst, 2)) self.assertEqual(user, utils.load_user(runtime_storage_inst, 'john_doe')) self.assertEqual(user, utils.load_user(runtime_storage_inst, '*****@*****.**')) # all records should have the same user_id and company name for record in runtime_storage_inst.get_all_records(): self.assertEqual('john_doe', record['user_id'], message='Record %s' % record['primary_key']) self.assertEqual('IBM', record['company_name'], message='Record %s' % record['primary_key'])
def test_process_blueprint_then_commit(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}, lp_info={'*****@*****.**': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890}, {'record_type': 'commit', 'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'date': 1234567890, 'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana'} ])) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[0]) self.assertRecordsMatch( {'record_type': 'commit', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'company_name': '*independent'}, processed_records[1]) user = {'seq': 1, 'core': [], 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def test_process_blueprint_one_draft_spawned_lp_knows_user(self): # In: blueprint record # LP knows user # Out: blueprint-draft record # new user profile created, name is taken from LP profile record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} ])) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[0]) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertEquals({ 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': [], 'companies': [{'company_name': '*independent', 'end_date': 0}] }, user)
def _update_members_company_name(self): LOG.debug('Update members with company names') for record in self.runtime_storage_inst.get_all_records(): if record['record_type'] != 'member': continue company_draft = record['company_draft'] company_name = self.domains_index.get( utils.normalize_company_name(company_draft)) or ( utils.normalize_company_draft(company_draft)) if company_name == record['company_name']: continue LOG.debug('Update record %s, company name changed to %s', record, company_name) record['company_name'] = company_name yield record user = utils.load_user(self.runtime_storage_inst, record['user_id']) LOG.debug('Update user %s, company name changed to %s', user, company_name) user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] utils.store_user(self.runtime_storage_inst, user)
def test_process_commit_existing_user_new_email_known_company_update(self): record_processor_inst = self.make_record_processor( users=[ {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} ], companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'*****@*****.**': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertIn('*****@*****.**', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name'], message='User affiliation should be updated')
def test_process_commit_existing_user_new_email_unknown_company(self): # User is known to LP, but his email is new to us. Should match # the user and return company from user profile record_processor_inst = self.make_record_processor( users=[ {'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{'company_name': 'NEC', 'end_date': 0}]} ], companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'*****@*****.**': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'NEC', } self.assertRecordsMatch(expected_commit, processed_commit) self.assertIn('*****@*****.**', utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')['emails'])
def _store_users(runtime_storage_inst, users): for user in users: stored_user = utils.load_user(runtime_storage_inst, user['user_id']) if stored_user: stored_user.update(user) user = stored_user utils.store_user(runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst): LOG.debug('Update company names for members') record_processor_inst = record_processor.RecordProcessor( runtime_storage_inst) member_iterator = _get_changed_member_records(runtime_storage_inst, record_processor_inst) for record in member_iterator: company_name = record['company_name'] user = utils.load_user(runtime_storage_inst, record['user_id']) user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name utils.store_user(runtime_storage_inst, user) LOG.debug('Company name changed for user %s', user) record_id = record['record_id'] runtime_storage_inst.memcached.set( runtime_storage_inst._get_record_name(record_id), record) runtime_storage_inst._commit_update(record_id)
def _process_member(self, record): user_id = "member:" + record["member_id"] record["primary_key"] = user_id record["date"] = utils.member_date_to_timestamp(record["date_joined"]) record["author_name"] = record["member_name"] record["module"] = "unknown" company_draft = record["company_draft"] company_name = self.domains_index.get(company_draft) or company_draft # author_email is a key to create new user record["author_email"] = user_id record["company_name"] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record["company_name"] = company_name user = utils.load_user(self.runtime_storage_inst, user_id) user["user_name"] = record["author_name"] user["companies"] = [{"company_name": company_name, "end_date": 0}] user["company_name"] = company_name utils.store_user(self.runtime_storage_inst, user) record["company_name"] = company_name yield record
def _process_member(self, record): user_id = "member:" + record['member_id'] record['primary_key'] = user_id record['date'] = utils.member_date_to_timestamp(record['date_joined']) record['author_name'] = record['member_name'] record['module'] = 'unknown' company_draft = record['company_draft'] company_name = self.domains_index.get(utils.normalize_company_name( company_draft)) or (utils.normalize_company_draft(company_draft)) # author_email is a key to create new user record['author_email'] = user_id record['company_name'] = company_name # _update_record_and_user function will create new user if needed self._update_record_and_user(record) record['company_name'] = company_name user = utils.load_user(self.runtime_storage_inst, user_id) user['user_name'] = record['author_name'] user['companies'] = [{ 'company_name': company_name, 'end_date': 0, }] user['company_name'] = company_name utils.store_user(self.runtime_storage_inst, user) record['company_name'] = company_name yield record
def test_process_commit_new_user_unknown_to_lb(self): # User is new to us and not known to LP # Should set user name and empty LPid record_processor_inst = self.make_record_processor( companies=[{ 'company_name': 'IBM', 'domains': ['ibm.com'] }]) processed_commit = list( record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': None, 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user(record_processor_inst.runtime_storage_inst, '*****@*****.**') self.assertIn('*****@*****.**', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name']) self.assertEquals(None, user['launchpad_id'])
def test_process_commit_new_user(self): # User is known to LP, but new to us # Should add new user and set company depending on email record_processor_inst = self.make_record_processor( companies=[{ 'company_name': 'IBM', 'domains': ['ibm.com'] }], lp_info={ '*****@*****.**': { 'name': 'john_doe', 'display_name': 'John Doe' } }) processed_commit = list( record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user(record_processor_inst.runtime_storage_inst, 'john_doe') self.assertIn('*****@*****.**', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name'])
def test_process_review_then_blueprint(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_records = list(record_processor_inst.process([ {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'John Doe', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1379404951, 'module': 'nova', 'branch': 'master'}, {'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890} ])) self.assertRecordsMatch( {'record_type': 'review', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'company_name': '*independent'}, processed_records[0]) self.assertRecordsMatch( {'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent'}, processed_records[1]) user = {'seq': 1, 'core': [], 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{'company_name': '*independent', 'end_date': 0}]} self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def test_process_commit_then_review_with_different_email(self): record_processor_inst = self.make_record_processor( lp_info={'*****@*****.**': {'name': 'john_doe', 'display_name': 'John Doe'}}, companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}]) list(record_processor_inst.process([ {'record_type': 'commit', 'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'date': 1234567890, 'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana'}, {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'Bill Smith', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1379404951, 'module': 'nova', 'branch': 'master', 'patchSets': [ {'number': '1', 'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76', 'ref': 'refs/changes/16/58516/1', 'uploader': {'name': 'Bill Smith', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1385470730, 'approvals': [ {'type': 'CRVW', 'description': 'Code Review', 'value': '1', 'grantedOn': 1385478464, 'by': {'name': 'John Doe', 'email': '*****@*****.**', 'username': '******'}}]}]} ])) user = {'seq': 1, 'core': [], 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**', '*****@*****.**'], 'companies': [{'company_name': 'IBM', 'end_date': 0}]} self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, '*****@*****.**')) self.assertEqual(user, utils.load_user( record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def test_process_commit_existing_user_new_email_known_company_update(self): record_processor_inst = self.make_record_processor( users=[{ 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{ 'company_name': '*independent', 'end_date': 0 }] }], companies=[{ 'company_name': 'IBM', 'domains': ['ibm.com'] }], lp_info={ '*****@*****.**': { 'name': 'john_doe', 'display_name': 'John Doe' } }) processed_commit = list( record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user(record_processor_inst.runtime_storage_inst, 'john_doe') self.assertIn('*****@*****.**', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name'], message='User affiliation should be updated')
def _update_members_company_name(runtime_storage_inst): LOG.debug("Update company names for members") record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst) member_iterator = _get_changed_member_records(runtime_storage_inst, record_processor_inst) for record in member_iterator: company_name = record["company_name"] user = utils.load_user(runtime_storage_inst, record["user_id"]) user["companies"] = [{"company_name": company_name, "end_date": 0}] user["company_name"] = company_name utils.store_user(runtime_storage_inst, user) LOG.debug("Company name changed for user %s", user) record_id = record["record_id"] runtime_storage_inst.memcached.set(runtime_storage_inst._get_record_name(record_id), record) runtime_storage_inst._commit_update(record_id)
def test_process_commit_existing_user_new_email_known_company(self): # User is known to LP, his email is new to us, and maps to other # company. Should return other company instead of those mentioned # in user profile record_processor_inst = self.make_record_processor( users=[{ 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{ 'company_name': 'NEC', 'end_date': 0 }] }], companies=[{ 'company_name': 'IBM', 'domains': ['ibm.com'] }], lp_info={ '*****@*****.**': { 'name': 'john_doe', 'display_name': 'John Doe' } }) processed_commit = list( record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) self.assertIn( '*****@*****.**', utils.load_user(record_processor_inst.runtime_storage_inst, 'john_doe')['emails'])
def test_process_blueprint_one_draft_spawned_lp_knows_user(self): # In: blueprint record # LP knows user # Out: blueprint-draft record # new user profile created, name is taken from LP profile record_processor_inst = self.make_record_processor(lp_user_name={ 'john_doe': { 'name': 'john_doe', 'display_name': 'John Doe' } }) processed_records = list( record_processor_inst.process([{ 'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890 }])) self.assertRecordsMatch( { 'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent' }, processed_records[0]) user = utils.load_user(record_processor_inst.runtime_storage_inst, 'john_doe') self.assertEquals( { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': [], 'companies': [{ 'company_name': '*independent', 'end_date': 0 }] }, user)
def test_process_commit_new_user_unknown_to_lb(self): # User is new to us and not known to LP # Should set user name and empty LPid record_processor_inst = self.make_record_processor( companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}]) processed_commit = list(record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': None, 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user( record_processor_inst.runtime_storage_inst, '*****@*****.**') self.assertIn('*****@*****.**', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name']) self.assertEquals(None, user['launchpad_id'])
def _update_records_with_user_info(self): LOG.debug('Update user info in records') for record in self.runtime_storage_inst.get_all_records(): need_update = False if record['user_id'] in self.updated_users: user = utils.load_user(self.runtime_storage_inst, record['user_id']) user_company_name = user['companies'][0]['company_name'] if record['company_name'] != user_company_name: LOG.debug('Update record %s: company changed to: %s', record['primary_key'], user_company_name) record['company_name'] = user_company_name need_update = True if record['user_id'] != user['user_id']: LOG.debug('Update record %s, user id changed to: %s', record['primary_key'], user['user_id']) record['user_id'] = user['user_id'] need_update = True if need_update: yield record
def test_process_commit_new_user(self): # User is known to LP, but new to us # Should add new user and set company depending on email record_processor_inst = self.make_record_processor( companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], lp_info={'*****@*****.**': {'name': 'john_doe', 'display_name': 'John Doe'}}) processed_commit = list(record_processor_inst.process( generate_commits(author_email='*****@*****.**', author_name='John Doe')))[0] expected_commit = { 'launchpad_id': 'john_doe', 'author_email': '*****@*****.**', 'author_name': 'John Doe', 'company_name': 'IBM', } self.assertRecordsMatch(expected_commit, processed_commit) user = utils.load_user( record_processor_inst.runtime_storage_inst, 'john_doe') self.assertIn('*****@*****.**', user['emails']) self.assertEquals('IBM', user['companies'][0]['company_name'])
def get_user_from_runtime_storage(user_id): runtime_storage_inst = get_vault()['runtime_storage'] return utils.load_user(runtime_storage_inst, user_id)
def test_core_user_guess(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'} }, companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}], ) runtime_storage_inst = record_processor_inst.runtime_storage_inst runtime_storage_inst.set_records(record_processor_inst.process([ {'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': {'name': 'John Doe', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1379404951, 'module': 'nova', 'branch': 'master', 'patchSets': [ {'number': '1', 'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76', 'ref': 'refs/changes/16/58516/1', 'uploader': { 'name': 'Bill Smith', 'email': '*****@*****.**', 'username': '******'}, 'createdOn': 1385470730, 'approvals': [ {'type': 'CRVW', 'description': 'Code Review', 'value': '2', 'grantedOn': 1385478464, 'by': { 'name': 'John Doe', 'email': '*****@*****.**', 'username': '******'}}, {'type': 'CRVW', 'description': 'Code Review', 'value': '-1', 'grantedOn': 1385478465, 'by': { 'name': 'Homer Simpson', 'email': '*****@*****.**', 'username': '******'}} ] }]} ])) record_processor_inst.update() user_1 = {'seq': 1, 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'core': [('nova', 'master')], 'companies': [{'company_name': 'IBM', 'end_date': 0}]} user_2 = {'seq': 2, 'user_id': 'homer', 'launchpad_id': 'homer', 'user_name': 'Homer Simpson', 'emails': ['*****@*****.**'], 'core': [], 'companies': [{'company_name': '*independent', 'end_date': 0}]} runtime_storage_inst = record_processor_inst.runtime_storage_inst self.assertEqual(user_1, utils.load_user(runtime_storage_inst, 'john_doe')) self.assertEqual(user_2, utils.load_user(runtime_storage_inst, 'homer'))
def _get_records_for_users_to_update(self): users_reviews = {} valid_blueprints = {} mentioned_blueprints = {} for record in self.runtime_storage_inst.get_all_records(): for bp in record.get('blueprint_id', []): if bp in mentioned_blueprints: mentioned_blueprints[bp]['count'] += 1 if record['date'] > mentioned_blueprints[bp]['date']: mentioned_blueprints[bp]['date'] = record['date'] else: mentioned_blueprints[bp] = { 'count': 1, 'date': record['date'] } if record['record_type'] in ['bpd', 'bpc']: valid_blueprints[record['id']] = { 'primary_key': record['primary_key'], 'count': 0, 'date': record['date'] } if record['record_type'] == 'review': launchpad_id = record['launchpad_id'] review = {'date': record['date'], 'id': record['id']} if launchpad_id in users_reviews: users_reviews[launchpad_id].append(review) else: users_reviews[launchpad_id] = [review] for bp_name, bp in valid_blueprints.iteritems(): if bp_name in mentioned_blueprints: bp['count'] = mentioned_blueprints[bp_name]['count'] bp['date'] = mentioned_blueprints[bp_name]['date'] else: bp['count'] = 0 bp['date'] = 0 reviews_index = {} for launchpad_id, reviews in users_reviews.iteritems(): reviews.sort(key=lambda x: x['date']) review_number = 0 for review in reviews: review_number += 1 review['review_number'] = review_number reviews_index[review['id']] = review for record in self.runtime_storage_inst.get_all_records(): need_update = False user_id = record['user_id'] if user_id in self.updated_users: user = utils.load_user(self.runtime_storage_inst, user_id) user_company_name = user['companies'][0]['company_name'] if record['company_name'] != user_company_name: LOG.debug('Update record %s: company changed to: %s', record['primary_key'], user_company_name) record['company_name'] = user_company_name need_update = True valid_bp = set([]) for bp in record.get('blueprint_id', []): if bp in valid_blueprints: valid_bp.add(bp) else: LOG.debug('Update record %s: removed invalid bp: %s', record['primary_key'], bp) need_update = True record['blueprint_id'] = list(valid_bp) if record['record_type'] in ['bpd', 'bpc']: bp = valid_blueprints[record['id']] if ((record.get('mention_count') != bp['count']) or (record.get('mention_date') != bp['date'])): record['mention_count'] = bp['count'] record['mention_date'] = bp['date'] LOG.debug('Update record %s: mention stats: (%s:%s)', record['primary_key'], bp['count'], bp['date']) need_update = True if record['record_type'] == 'review': review = reviews_index[record['id']] if record.get('review_number') != review['review_number']: record['review_number'] = review['review_number'] need_update = True if need_update: yield record
def test_process_review_then_blueprint(self): record_processor_inst = self.make_record_processor(lp_user_name={ 'john_doe': { 'name': 'john_doe', 'display_name': 'John Doe' } }) processed_records = list( record_processor_inst.process([{ 'record_type': 'review', 'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e', 'subject': 'Fix AttributeError in Keypair._add_details()', 'owner': { 'name': 'John Doe', 'email': '*****@*****.**', 'username': '******' }, 'createdOn': 1379404951, 'module': 'nova' }, { 'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890 }])) self.assertRecordsMatch( { 'record_type': 'review', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'company_name': '*independent' }, processed_records[0]) self.assertRecordsMatch( { 'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent' }, processed_records[1]) user = { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{ 'company_name': '*independent', 'end_date': 0 }] } self.assertEquals( user, utils.load_user(record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEquals( user, utils.load_user(record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def get_user_from_runtime_storage(user_id): runtime_storage_inst = get_vault()['runtime_storage'] user_index = get_vault()['user_index'] if user_id not in user_index: user_index[user_id] = utils.load_user(runtime_storage_inst, user_id) return user_index[user_id]
def test_process_blueprint_then_commit(self): record_processor_inst = self.make_record_processor( lp_user_name={ 'john_doe': { 'name': 'john_doe', 'display_name': 'John Doe' } }, lp_info={ '*****@*****.**': { 'name': 'john_doe', 'display_name': 'John Doe' } }) processed_records = list( record_processor_inst.process([{ 'record_type': 'bp', 'id': 'mod:blueprint', 'self_link': 'http://launchpad.net/blueprint', 'owner': 'john_doe', 'date_created': 1234567890 }, { 'record_type': 'commit', 'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'date': 1234567890, 'lines_added': 25, 'lines_deleted': 9, 'release_name': 'havana' }])) self.assertRecordsMatch( { 'record_type': 'bpd', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'company_name': '*independent' }, processed_records[0]) self.assertRecordsMatch( { 'record_type': 'commit', 'launchpad_id': 'john_doe', 'author_name': 'John Doe', 'author_email': '*****@*****.**', 'company_name': '*independent' }, processed_records[1]) user = { 'user_id': 'john_doe', 'launchpad_id': 'john_doe', 'user_name': 'John Doe', 'emails': ['*****@*****.**'], 'companies': [{ 'company_name': '*independent', 'end_date': 0 }] } self.assertEquals( user, utils.load_user(record_processor_inst.runtime_storage_inst, 'john_doe')) self.assertEquals( user, utils.load_user(record_processor_inst.runtime_storage_inst, '*****@*****.**'))