def update_user(self, record):
email = record.get('author_email')
user_e = utils.load_user(self.runtime_storage_inst, email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if email and (not user_e) and (not launchpad_id):
# query LP
launchpad_id, lp_user_name = self._get_lp_info(email)
if lp_user_name:
user_name = lp_user_name
user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {}
user = self._create_user(launchpad_id, email, user_name)
if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'):
# sequence numbers are set and the same, merge is not needed
user = user_e
else:
if user_e or user_l:
user = self._merge_user_profiles(user_e, user_l, user)
else:
# create new
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
utils.store_user(self.runtime_storage_inst, user)
return user
def update_user(self, record):
email = record.get('author_email')
user = utils.load_user(self.runtime_storage_inst, email)
if not user:
if record.get('launchpad_id'):
launchpad_id = record.get('launchpad_id')
user_name = record.get('author_name')
else:
launchpad_id, user_name = self._get_lp_info(email)
if launchpad_id:
user = utils.load_user(self.runtime_storage_inst, launchpad_id)
if user:
# merge emails
if email:
self._update_user_profile(user, email)
else:
# create new
if not user_name:
user_name = record.get('author_name')
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
user = self._create_user(launchpad_id, email, user_name)
utils.store_user(self.runtime_storage_inst, user)
return user
def update_user(self, record):
email = record.get('author_email')
user_e = utils.load_user(self.runtime_storage_inst, email) or {}
user_name = record.get('author_name')
launchpad_id = record.get('launchpad_id')
if email and (not user_e) and (not launchpad_id):
# query LP
launchpad_id, lp_user_name = self._get_lp_info(email)
if lp_user_name:
user_name = lp_user_name
user_l = utils.load_user(self.runtime_storage_inst, launchpad_id) or {}
user = self._create_user(launchpad_id, email, user_name)
if (user_e.get('seq') == user_l.get('seq')) and user_e.get('seq'):
# sequence numbers are set and the same, merge is not needed
user = user_e
else:
if user_e or user_l:
user = self._merge_user_profiles([user_e, user_l, user])
else:
# create new
if not user_name:
user_name = self._get_lp_user_name(launchpad_id)
if user_name:
user['user_name'] = user_name
LOG.debug('Created new user: %s', user)
utils.store_user(self.runtime_storage_inst, user)
return user
def test_process_email_then_review(self):
# it is expected that the user profile will contain both email and
# LP id
record_processor_inst = self.make_record_processor()
list(record_processor_inst.process([
{'record_type': 'email',
'message_id': '<message-id>',
'author_email': '*****@*****.**',
'subject': 'hello, world!',
'body': 'lorem ipsum',
'date': 1234567890},
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': '*****@*****.**',
'username': '******'},
'createdOn': 1379404951,
'module': 'nova', 'branch': 'master'}
]))
user = {'seq': 1,
'core': [],
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{'company_name': '*independent', 'end_date': 0}]}
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, '*****@*****.**'))
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe'))
def test_merge_users(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}
},
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
)
runtime_storage_inst = record_processor_inst.runtime_storage_inst
runtime_storage_inst.set_records(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890},
{'record_type': 'email',
'message_id': '<message-id>',
'author_email': '*****@*****.**', 'author_name': 'John Doe',
'subject': 'hello, world!',
'body': 'lorem ipsum',
'date': 1234567890},
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': '*****@*****.**',
'username': '******'},
'createdOn': 1379404951,
'module': 'nova', 'branch': 'master'}
]))
record_processor_inst.update()
user = {'seq': 2,
'core': [],
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{'company_name': 'IBM', 'end_date': 0}]}
runtime_storage_inst = record_processor_inst.runtime_storage_inst
self.assertEqual(2, runtime_storage_inst.get_by_key('user:count'))
self.assertEqual(None, utils.load_user(runtime_storage_inst, 1))
self.assertEqual(user, utils.load_user(runtime_storage_inst, 2))
self.assertEqual(user, utils.load_user(runtime_storage_inst,
'john_doe'))
self.assertEqual(user, utils.load_user(runtime_storage_inst,
'*****@*****.**'))
# all records should have the same user_id and company name
for record in runtime_storage_inst.get_all_records():
self.assertEqual('john_doe', record['user_id'],
message='Record %s' % record['primary_key'])
self.assertEqual('IBM', record['company_name'],
message='Record %s' % record['primary_key'])
def test_process_blueprint_then_commit(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}},
lp_info={'*****@*****.**':
{'name': 'john_doe', 'display_name': 'John Doe'}})
processed_records = list(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890},
{'record_type': 'commit',
'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
'author_name': 'John Doe',
'author_email': '*****@*****.**',
'date': 1234567890,
'lines_added': 25,
'lines_deleted': 9,
'release_name': 'havana'}
]))
self.assertRecordsMatch(
{'record_type': 'bpd',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'company_name': '*independent'},
processed_records[0])
self.assertRecordsMatch(
{'record_type': 'commit',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'author_email': '*****@*****.**',
'company_name': '*independent'},
processed_records[1])
user = {'seq': 1,
'core': [],
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{'company_name': '*independent', 'end_date': 0}]}
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe'))
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def test_process_blueprint_one_draft_spawned_lp_knows_user(self):
# In: blueprint record
# LP knows user
# Out: blueprint-draft record
# new user profile created, name is taken from LP profile
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
processed_records = list(record_processor_inst.process([
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890}
]))
self.assertRecordsMatch(
{'record_type': 'bpd',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'company_name': '*independent'},
processed_records[0])
user = utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe')
self.assertEquals({
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': [],
'companies': [{'company_name': '*independent', 'end_date': 0}]
}, user)
def _update_members_company_name(self):
LOG.debug('Update members with company names')
for record in self.runtime_storage_inst.get_all_records():
if record['record_type'] != 'member':
continue
company_draft = record['company_draft']
company_name = self.domains_index.get(
utils.normalize_company_name(company_draft)) or (
utils.normalize_company_draft(company_draft))
if company_name == record['company_name']:
continue
LOG.debug('Update record %s, company name changed to %s',
record, company_name)
record['company_name'] = company_name
yield record
user = utils.load_user(self.runtime_storage_inst,
record['user_id'])
LOG.debug('Update user %s, company name changed to %s',
user, company_name)
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
utils.store_user(self.runtime_storage_inst, user)
def test_process_commit_existing_user_new_email_known_company_update(self):
record_processor_inst = self.make_record_processor(
users=[
{'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{'company_name': '*independent',
'end_date': 0}]}
],
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
lp_info={'*****@*****.**':
{'name': 'john_doe', 'display_name': 'John Doe'}})
processed_commit = list(record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': 'john_doe',
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe')
self.assertIn('*****@*****.**', user['emails'])
self.assertEquals('IBM', user['companies'][0]['company_name'],
message='User affiliation should be updated')
def test_process_commit_existing_user_new_email_unknown_company(self):
# User is known to LP, but his email is new to us. Should match
# the user and return company from user profile
record_processor_inst = self.make_record_processor(
users=[
{'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{'company_name': 'NEC', 'end_date': 0}]}
],
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
lp_info={'*****@*****.**':
{'name': 'john_doe', 'display_name': 'John Doe'}})
processed_commit = list(record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': 'john_doe',
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'NEC',
}
self.assertRecordsMatch(expected_commit, processed_commit)
self.assertIn('*****@*****.**', utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe')['emails'])
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = utils.load_user(runtime_storage_inst, user['user_id'])
if stored_user:
stored_user.update(user)
user = stored_user
utils.store_user(runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst):
LOG.debug('Update company names for members')
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
member_iterator = _get_changed_member_records(runtime_storage_inst,
record_processor_inst)
for record in member_iterator:
company_name = record['company_name']
user = utils.load_user(runtime_storage_inst, record['user_id'])
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
utils.store_user(runtime_storage_inst, user)
LOG.debug('Company name changed for user %s', user)
record_id = record['record_id']
runtime_storage_inst.memcached.set(
runtime_storage_inst._get_record_name(record_id), record)
runtime_storage_inst._commit_update(record_id)
def _store_users(runtime_storage_inst, users):
for user in users:
stored_user = utils.load_user(runtime_storage_inst, user['user_id'])
if stored_user:
stored_user.update(user)
user = stored_user
utils.store_user(runtime_storage_inst, user)
def _update_members_company_name(runtime_storage_inst):
LOG.debug('Update company names for members')
record_processor_inst = record_processor.RecordProcessor(
runtime_storage_inst)
member_iterator = _get_changed_member_records(runtime_storage_inst,
record_processor_inst)
for record in member_iterator:
company_name = record['company_name']
user = utils.load_user(runtime_storage_inst, record['user_id'])
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
utils.store_user(runtime_storage_inst, user)
LOG.debug('Company name changed for user %s', user)
record_id = record['record_id']
runtime_storage_inst.memcached.set(
runtime_storage_inst._get_record_name(record_id), record)
runtime_storage_inst._commit_update(record_id)
def _process_member(self, record):
user_id = "member:" + record["member_id"]
record["primary_key"] = user_id
record["date"] = utils.member_date_to_timestamp(record["date_joined"])
record["author_name"] = record["member_name"]
record["module"] = "unknown"
company_draft = record["company_draft"]
company_name = self.domains_index.get(company_draft) or company_draft
# author_email is a key to create new user
record["author_email"] = user_id
record["company_name"] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record["company_name"] = company_name
user = utils.load_user(self.runtime_storage_inst, user_id)
user["user_name"] = record["author_name"]
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user["company_name"] = company_name
utils.store_user(self.runtime_storage_inst, user)
record["company_name"] = company_name
yield record
def _process_member(self, record):
user_id = "member:" + record['member_id']
record['primary_key'] = user_id
record['date'] = utils.member_date_to_timestamp(record['date_joined'])
record['author_name'] = record['member_name']
record['module'] = 'unknown'
company_draft = record['company_draft']
company_name = self.domains_index.get(utils.normalize_company_name(
company_draft)) or (utils.normalize_company_draft(company_draft))
# author_email is a key to create new user
record['author_email'] = user_id
record['company_name'] = company_name
# _update_record_and_user function will create new user if needed
self._update_record_and_user(record)
record['company_name'] = company_name
user = utils.load_user(self.runtime_storage_inst, user_id)
user['user_name'] = record['author_name']
user['companies'] = [{
'company_name': company_name,
'end_date': 0,
}]
user['company_name'] = company_name
utils.store_user(self.runtime_storage_inst, user)
record['company_name'] = company_name
yield record
def test_process_commit_new_user_unknown_to_lb(self):
# User is new to us and not known to LP
# Should set user name and empty LPid
record_processor_inst = self.make_record_processor(
companies=[{
'company_name': 'IBM',
'domains': ['ibm.com']
}])
processed_commit = list(
record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': None,
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = utils.load_user(record_processor_inst.runtime_storage_inst,
'*****@*****.**')
self.assertIn('*****@*****.**', user['emails'])
self.assertEquals('IBM', user['companies'][0]['company_name'])
self.assertEquals(None, user['launchpad_id'])
def test_process_commit_new_user(self):
# User is known to LP, but new to us
# Should add new user and set company depending on email
record_processor_inst = self.make_record_processor(
companies=[{
'company_name': 'IBM',
'domains': ['ibm.com']
}],
lp_info={
'*****@*****.**': {
'name': 'john_doe',
'display_name': 'John Doe'
}
})
processed_commit = list(
record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': 'john_doe',
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = utils.load_user(record_processor_inst.runtime_storage_inst,
'john_doe')
self.assertIn('*****@*****.**', user['emails'])
self.assertEquals('IBM', user['companies'][0]['company_name'])
def test_process_review_then_blueprint(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}})
processed_records = list(record_processor_inst.process([
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': '*****@*****.**',
'username': '******'},
'createdOn': 1379404951,
'module': 'nova', 'branch': 'master'},
{'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890}
]))
self.assertRecordsMatch(
{'record_type': 'review',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'author_email': '*****@*****.**',
'company_name': '*independent'},
processed_records[0])
self.assertRecordsMatch(
{'record_type': 'bpd',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'company_name': '*independent'},
processed_records[1])
user = {'seq': 1,
'core': [],
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{'company_name': '*independent', 'end_date': 0}]}
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe'))
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def test_process_commit_then_review_with_different_email(self):
record_processor_inst = self.make_record_processor(
lp_info={'*****@*****.**':
{'name': 'john_doe', 'display_name': 'John Doe'}},
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}])
list(record_processor_inst.process([
{'record_type': 'commit',
'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
'author_name': 'John Doe', 'author_email': '*****@*****.**',
'date': 1234567890, 'lines_added': 25, 'lines_deleted': 9,
'release_name': 'havana'},
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'Bill Smith', 'email': '*****@*****.**',
'username': '******'},
'createdOn': 1379404951, 'module': 'nova', 'branch': 'master',
'patchSets': [
{'number': '1',
'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76',
'ref': 'refs/changes/16/58516/1',
'uploader': {'name': 'Bill Smith', 'email': '*****@*****.**',
'username': '******'},
'createdOn': 1385470730,
'approvals': [
{'type': 'CRVW', 'description': 'Code Review',
'value': '1', 'grantedOn': 1385478464,
'by': {'name': 'John Doe', 'email': '*****@*****.**',
'username': '******'}}]}]}
]))
user = {'seq': 1,
'core': [],
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**', '*****@*****.**'],
'companies': [{'company_name': 'IBM', 'end_date': 0}]}
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe'))
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, '*****@*****.**'))
self.assertEqual(user, utils.load_user(
record_processor_inst.runtime_storage_inst, '*****@*****.**'))
def test_process_commit_existing_user_new_email_known_company_update(self):
record_processor_inst = self.make_record_processor(
users=[{
'user_id':
'john_doe',
'launchpad_id':
'john_doe',
'user_name':
'John Doe',
'emails': ['*****@*****.**'],
'companies': [{
'company_name': '*independent',
'end_date': 0
}]
}],
companies=[{
'company_name': 'IBM',
'domains': ['ibm.com']
}],
lp_info={
'*****@*****.**': {
'name': 'john_doe',
'display_name': 'John Doe'
}
})
processed_commit = list(
record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': 'john_doe',
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = utils.load_user(record_processor_inst.runtime_storage_inst,
'john_doe')
self.assertIn('*****@*****.**', user['emails'])
self.assertEquals('IBM',
user['companies'][0]['company_name'],
message='User affiliation should be updated')
def _update_members_company_name(runtime_storage_inst):
LOG.debug("Update company names for members")
record_processor_inst = record_processor.RecordProcessor(runtime_storage_inst)
member_iterator = _get_changed_member_records(runtime_storage_inst, record_processor_inst)
for record in member_iterator:
company_name = record["company_name"]
user = utils.load_user(runtime_storage_inst, record["user_id"])
user["companies"] = [{"company_name": company_name, "end_date": 0}]
user["company_name"] = company_name
utils.store_user(runtime_storage_inst, user)
LOG.debug("Company name changed for user %s", user)
record_id = record["record_id"]
runtime_storage_inst.memcached.set(runtime_storage_inst._get_record_name(record_id), record)
runtime_storage_inst._commit_update(record_id)
def test_process_commit_existing_user_new_email_known_company(self):
# User is known to LP, his email is new to us, and maps to other
# company. Should return other company instead of those mentioned
# in user profile
record_processor_inst = self.make_record_processor(
users=[{
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{
'company_name': 'NEC',
'end_date': 0
}]
}],
companies=[{
'company_name': 'IBM',
'domains': ['ibm.com']
}],
lp_info={
'*****@*****.**': {
'name': 'john_doe',
'display_name': 'John Doe'
}
})
processed_commit = list(
record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': 'john_doe',
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
self.assertIn(
'*****@*****.**',
utils.load_user(record_processor_inst.runtime_storage_inst,
'john_doe')['emails'])
def test_process_blueprint_one_draft_spawned_lp_knows_user(self):
# In: blueprint record
# LP knows user
# Out: blueprint-draft record
# new user profile created, name is taken from LP profile
record_processor_inst = self.make_record_processor(lp_user_name={
'john_doe': {
'name': 'john_doe',
'display_name': 'John Doe'
}
})
processed_records = list(
record_processor_inst.process([{
'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890
}]))
self.assertRecordsMatch(
{
'record_type': 'bpd',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'company_name': '*independent'
}, processed_records[0])
user = utils.load_user(record_processor_inst.runtime_storage_inst,
'john_doe')
self.assertEquals(
{
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': [],
'companies': [{
'company_name': '*independent',
'end_date': 0
}]
}, user)
def test_process_commit_new_user_unknown_to_lb(self):
# User is new to us and not known to LP
# Should set user name and empty LPid
record_processor_inst = self.make_record_processor(
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}])
processed_commit = list(record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': None,
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = utils.load_user(
record_processor_inst.runtime_storage_inst, '*****@*****.**')
self.assertIn('*****@*****.**', user['emails'])
self.assertEquals('IBM', user['companies'][0]['company_name'])
self.assertEquals(None, user['launchpad_id'])
def _update_records_with_user_info(self):
LOG.debug('Update user info in records')
for record in self.runtime_storage_inst.get_all_records():
need_update = False
if record['user_id'] in self.updated_users:
user = utils.load_user(self.runtime_storage_inst,
record['user_id'])
user_company_name = user['companies'][0]['company_name']
if record['company_name'] != user_company_name:
LOG.debug('Update record %s: company changed to: %s',
record['primary_key'], user_company_name)
record['company_name'] = user_company_name
need_update = True
if record['user_id'] != user['user_id']:
LOG.debug('Update record %s, user id changed to: %s',
record['primary_key'], user['user_id'])
record['user_id'] = user['user_id']
need_update = True
if need_update:
yield record
def test_process_commit_new_user(self):
# User is known to LP, but new to us
# Should add new user and set company depending on email
record_processor_inst = self.make_record_processor(
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
lp_info={'*****@*****.**':
{'name': 'john_doe', 'display_name': 'John Doe'}})
processed_commit = list(record_processor_inst.process(
generate_commits(author_email='*****@*****.**',
author_name='John Doe')))[0]
expected_commit = {
'launchpad_id': 'john_doe',
'author_email': '*****@*****.**',
'author_name': 'John Doe',
'company_name': 'IBM',
}
self.assertRecordsMatch(expected_commit, processed_commit)
user = utils.load_user(
record_processor_inst.runtime_storage_inst, 'john_doe')
self.assertIn('*****@*****.**', user['emails'])
self.assertEquals('IBM', user['companies'][0]['company_name'])
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
return utils.load_user(runtime_storage_inst, user_id)
def test_core_user_guess(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {'name': 'john_doe', 'display_name': 'John Doe'}
},
companies=[{'company_name': 'IBM', 'domains': ['ibm.com']}],
)
runtime_storage_inst = record_processor_inst.runtime_storage_inst
runtime_storage_inst.set_records(record_processor_inst.process([
{'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {'name': 'John Doe',
'email': '*****@*****.**',
'username': '******'},
'createdOn': 1379404951,
'module': 'nova',
'branch': 'master',
'patchSets': [
{'number': '1',
'revision': '4d8984e92910c37b7d101c1ae8c8283a2e6f4a76',
'ref': 'refs/changes/16/58516/1',
'uploader': {
'name': 'Bill Smith',
'email': '*****@*****.**',
'username': '******'},
'createdOn': 1385470730,
'approvals': [
{'type': 'CRVW', 'description': 'Code Review',
'value': '2', 'grantedOn': 1385478464,
'by': {
'name': 'John Doe',
'email': '*****@*****.**',
'username': '******'}},
{'type': 'CRVW', 'description': 'Code Review',
'value': '-1', 'grantedOn': 1385478465,
'by': {
'name': 'Homer Simpson',
'email': '*****@*****.**',
'username': '******'}}
]
}]}
]))
record_processor_inst.update()
user_1 = {'seq': 1, 'user_id': 'john_doe',
'launchpad_id': 'john_doe', 'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'core': [('nova', 'master')],
'companies': [{'company_name': 'IBM', 'end_date': 0}]}
user_2 = {'seq': 2, 'user_id': 'homer',
'launchpad_id': 'homer', 'user_name': 'Homer Simpson',
'emails': ['*****@*****.**'],
'core': [],
'companies': [{'company_name': '*independent',
'end_date': 0}]}
runtime_storage_inst = record_processor_inst.runtime_storage_inst
self.assertEqual(user_1, utils.load_user(runtime_storage_inst,
'john_doe'))
self.assertEqual(user_2, utils.load_user(runtime_storage_inst,
'homer'))
def _get_records_for_users_to_update(self):
users_reviews = {}
valid_blueprints = {}
mentioned_blueprints = {}
for record in self.runtime_storage_inst.get_all_records():
for bp in record.get('blueprint_id', []):
if bp in mentioned_blueprints:
mentioned_blueprints[bp]['count'] += 1
if record['date'] > mentioned_blueprints[bp]['date']:
mentioned_blueprints[bp]['date'] = record['date']
else:
mentioned_blueprints[bp] = {
'count': 1,
'date': record['date']
}
if record['record_type'] in ['bpd', 'bpc']:
valid_blueprints[record['id']] = {
'primary_key': record['primary_key'],
'count': 0,
'date': record['date']
}
if record['record_type'] == 'review':
launchpad_id = record['launchpad_id']
review = {'date': record['date'], 'id': record['id']}
if launchpad_id in users_reviews:
users_reviews[launchpad_id].append(review)
else:
users_reviews[launchpad_id] = [review]
for bp_name, bp in valid_blueprints.iteritems():
if bp_name in mentioned_blueprints:
bp['count'] = mentioned_blueprints[bp_name]['count']
bp['date'] = mentioned_blueprints[bp_name]['date']
else:
bp['count'] = 0
bp['date'] = 0
reviews_index = {}
for launchpad_id, reviews in users_reviews.iteritems():
reviews.sort(key=lambda x: x['date'])
review_number = 0
for review in reviews:
review_number += 1
review['review_number'] = review_number
reviews_index[review['id']] = review
for record in self.runtime_storage_inst.get_all_records():
need_update = False
user_id = record['user_id']
if user_id in self.updated_users:
user = utils.load_user(self.runtime_storage_inst, user_id)
user_company_name = user['companies'][0]['company_name']
if record['company_name'] != user_company_name:
LOG.debug('Update record %s: company changed to: %s',
record['primary_key'], user_company_name)
record['company_name'] = user_company_name
need_update = True
valid_bp = set([])
for bp in record.get('blueprint_id', []):
if bp in valid_blueprints:
valid_bp.add(bp)
else:
LOG.debug('Update record %s: removed invalid bp: %s',
record['primary_key'], bp)
need_update = True
record['blueprint_id'] = list(valid_bp)
if record['record_type'] in ['bpd', 'bpc']:
bp = valid_blueprints[record['id']]
if ((record.get('mention_count') != bp['count']) or
(record.get('mention_date') != bp['date'])):
record['mention_count'] = bp['count']
record['mention_date'] = bp['date']
LOG.debug('Update record %s: mention stats: (%s:%s)',
record['primary_key'], bp['count'], bp['date'])
need_update = True
if record['record_type'] == 'review':
review = reviews_index[record['id']]
if record.get('review_number') != review['review_number']:
record['review_number'] = review['review_number']
need_update = True
if need_update:
yield record
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
return utils.load_user(runtime_storage_inst, user_id)
def test_process_review_then_blueprint(self):
record_processor_inst = self.make_record_processor(lp_user_name={
'john_doe': {
'name': 'john_doe',
'display_name': 'John Doe'
}
})
processed_records = list(
record_processor_inst.process([{
'record_type': 'review',
'id': 'I1045730e47e9e6ad31fcdfbaefdad77e2f3b2c3e',
'subject': 'Fix AttributeError in Keypair._add_details()',
'owner': {
'name': 'John Doe',
'email': '*****@*****.**',
'username': '******'
},
'createdOn': 1379404951,
'module': 'nova'
}, {
'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890
}]))
self.assertRecordsMatch(
{
'record_type': 'review',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'author_email': '*****@*****.**',
'company_name': '*independent'
}, processed_records[0])
self.assertRecordsMatch(
{
'record_type': 'bpd',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'company_name': '*independent'
}, processed_records[1])
user = {
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{
'company_name': '*independent',
'end_date': 0
}]
}
self.assertEquals(
user,
utils.load_user(record_processor_inst.runtime_storage_inst,
'john_doe'))
self.assertEquals(
user,
utils.load_user(record_processor_inst.runtime_storage_inst,
'*****@*****.**'))
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
user_index = get_vault()['user_index']
if user_id not in user_index:
user_index[user_id] = utils.load_user(runtime_storage_inst, user_id)
return user_index[user_id]
def get_user_from_runtime_storage(user_id):
runtime_storage_inst = get_vault()['runtime_storage']
user_index = get_vault()['user_index']
if user_id not in user_index:
user_index[user_id] = utils.load_user(runtime_storage_inst, user_id)
return user_index[user_id]
def test_process_blueprint_then_commit(self):
record_processor_inst = self.make_record_processor(
lp_user_name={
'john_doe': {
'name': 'john_doe',
'display_name': 'John Doe'
}
},
lp_info={
'*****@*****.**': {
'name': 'john_doe',
'display_name': 'John Doe'
}
})
processed_records = list(
record_processor_inst.process([{
'record_type': 'bp',
'id': 'mod:blueprint',
'self_link': 'http://launchpad.net/blueprint',
'owner': 'john_doe',
'date_created': 1234567890
}, {
'record_type': 'commit',
'commit_id': 'de7e8f297c193fb310f22815334a54b9c76a0be1',
'author_name': 'John Doe',
'author_email': '*****@*****.**',
'date': 1234567890,
'lines_added': 25,
'lines_deleted': 9,
'release_name': 'havana'
}]))
self.assertRecordsMatch(
{
'record_type': 'bpd',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'company_name': '*independent'
}, processed_records[0])
self.assertRecordsMatch(
{
'record_type': 'commit',
'launchpad_id': 'john_doe',
'author_name': 'John Doe',
'author_email': '*****@*****.**',
'company_name': '*independent'
}, processed_records[1])
user = {
'user_id': 'john_doe',
'launchpad_id': 'john_doe',
'user_name': 'John Doe',
'emails': ['*****@*****.**'],
'companies': [{
'company_name': '*independent',
'end_date': 0
}]
}
self.assertEquals(
user,
utils.load_user(record_processor_inst.runtime_storage_inst,
'john_doe'))
self.assertEquals(
user,
utils.load_user(record_processor_inst.runtime_storage_inst,
'*****@*****.**'))