def setUp(self):
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue1.json", 'r', encoding='utf-8') as \
issue_1_file:
self.issue_1 = json.load(issue_1_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue95.json", 'r', encoding='utf-8') as \
issue_95_file:
self.issue_95 = json.load(issue_95_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue95_comments.json", 'r', encoding='utf-8') as \
issue_95_comments_file:
self.issue_95_comments = json.load(issue_95_comments_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue95_history.json", 'r', encoding='utf-8') as \
issue_95_history_file:
self.issue_95_history = json.load(issue_95_history_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/conor_apache_org_user.json", 'r', encoding='utf-8') as \
conor_user_file:
self.conor_user = json.load(conor_user_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/dev_tomcat_apache_org_user.json", 'r', encoding='utf-8') as \
dev_tomcat_file:
self.dev_tomcat_file = json.load(dev_tomcat_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/craig_mcclanahan_user.json", 'r', encoding='utf-8') as \
craig_user_file:
self.craig_user = json.load(craig_user_file)
# Create testconfig
config = configparser.ConfigParser()
config.read(
os.path.dirname(os.path.realpath(__file__)) +
"/data/used_test_config.cfg")
# Setting up database with data that is normally put into it via vcs program
connect(config['Database']['db_database'],
username=config['Database']['db_user'],
password=config['Database']['db_password'],
host=config['Database']['db_hostname'],
port=int(config['Database']['db_port']),
authentication_source=config['Database']['db_authentication'],
connect=False)
Project.drop_collection()
IssueSystem.drop_collection()
Issue.drop_collection()
IssueComment.drop_collection()
Event.drop_collection()
self.project_id = Project(name='Bla').save().id
self.issues_system_id = IssueSystem(
project_id=self.project_id,
url="https://issues.apache.org/search?jql=project=BLA",
last_updated=datetime.datetime.now()).save().id
self.conf = ConfigMock(None, None, None, None, None, None, 'Bla',
'Nonsense?product=Blub', 'bugzilla', None, None,
None, None, None, None, 'DEBUG', '123')
def setUp(self):
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/people.json", 'r', encoding='utf-8') as people_file:
self.person = json.load(people_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6131.json", 'r', encoding='utf-8') as issues_file:
self.issue_6131 = json.load(issues_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6131_events.json", 'r', encoding='utf-8') as event_file:
self.events_issue_6131 = json.load(event_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6131_comments.json", 'r', encoding='utf-8') as cmt_file:
self.comments_issue_6131 = json.load(cmt_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6050.json", 'r', encoding='utf-8') as issues_file:
self.issue_6050 = json.load(issues_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6050_events.json", 'r', encoding='utf-8') as event_file:
self.events_issue_6050 = json.load(event_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6050_comments.json", 'r', encoding='utf-8') as cmt_file:
self.comments_issue_6050 = json.load(cmt_file)
# Create testconfig
config = configparser.ConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__)) + "/data/used_test_config.cfg")
# Setting up database with data that is normally put into it via vcs program
connect(config['Database']['db_database'], username=config['Database']['db_user'],
password=config['Database']['db_password'], host=config['Database']['db_hostname'],
port=int(config['Database']['db_port']),
authentication_source=config['Database']['db_authentication'],
connect=False)
Project.drop_collection()
IssueSystem.drop_collection()
Issue.drop_collection()
IssueComment.drop_collection()
Event.drop_collection()
self.project_id = Project(name='Composer').save().id
self.issues_system_id = IssueSystem(project_id=self.project_id, url="http://blub.de",
last_updated=datetime.datetime.now()).save().id
self.conf = ConfigMock(None, None, None, None, None, None, 'Ant', 'http://blub.de', 'github', None, None, None,
None, None, None, 'DEBUG', '123')
def _process_event(self, unique_event_id, bz_event, mongo_issue, change_date, author_id):
"""
Processes the event. During the event processing the Issue is set back to its original state
before the event occured.
:param unique_event_id: unique identifier of the event
:param bz_event: event that was received from the bugzilla API
:param mongo_issue: issue that is/should be stored in the mongodb
:param change_date: date when the event was created
:param author_id: :class:`bson.objectid.ObjectId` of the author of the event
"""
is_new_event = True
try:
mongo_event = Event.objects(external_id=unique_event_id, issue_id=mongo_issue.id).get()
is_new_event = False
except DoesNotExist:
mongo_event = Event(
external_id=unique_event_id,
issue_id=mongo_issue.id,
created_at=change_date,
author_id=author_id
)
# We need to map back the status from the bz terminology to ours. Special: The assigned_to must be mapped to
# assigned_to_detail beforehand, as we are using this for the issue parsing
if bz_event['field_name'] == 'assigned_to':
bz_at_name = 'assigned_to_detail'
else:
bz_at_name = bz_event['field_name']
try:
mongo_event.status = self.at_mapping[bz_at_name]
except KeyError:
logger.warning('Mapping for attribute %s not found.' % bz_at_name)
mongo_event.status = bz_at_name
# Check if the mongo_issue has the attribute.
# If yes: We can use the mongo_issue to set the old and new value of the event
# If no: We use the added / removed fields
if hasattr(mongo_issue, mongo_event.status):
mongo_event.new_value = copy.deepcopy(getattr(mongo_issue, mongo_event.status))
self._set_back_mongo_issue(mongo_issue, mongo_event.status, bz_event)
mongo_event.old_value = copy.deepcopy(getattr(mongo_issue, mongo_event.status))
else:
mongo_event.new_value = bz_event['added']
mongo_event.old_value = bz_event['removed']
return mongo_event, is_new_event
def test_store_events_two_times(self):
new_jira_backend = JiraBackend(self.conf, self.issues_system_id,
self.project_id)
issue = jira.resources.Issue(options=None,
session=None,
raw=self.issue_drill_1)
mongo_issue = Issue(external_id="TEST",
issue_system_id=self.issues_system_id).save()
new_jira_backend._store_events(issue, mongo_issue.id)
new_jira_backend._store_events(issue, mongo_issue.id)
stored_events = Event.objects(issue_id=mongo_issue.id).all()
self.assertEqual(7, len(stored_events))
def setUp(self):
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/drill_1_issue.json",
'r',
encoding='utf-8') as drill_1:
self.issue_drill_1 = json.load(drill_1)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/drill_138_issue.json",
'r',
encoding='utf-8') as drill_138:
self.issue_drill_138 = json.load(drill_138)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/drill_38_issue.json",
'r',
encoding='utf-8') as drill_38:
self.issue_drill_38 = json.load(drill_38)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/get_user1.json",
'r',
encoding='utf-8') as user1_file:
self.user1 = json.load(user1_file)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/get_user2.json",
'r',
encoding='utf-8') as user2_file:
self.user2 = json.load(user2_file)
# Create testconfig
config = configparser.ConfigParser()
config.read(
os.path.dirname(os.path.realpath(__file__)) +
"/data/used_test_config.cfg")
# Setting up database with data that is normally put into it via vcs program
connect(config['Database']['db_database'],
username=config['Database']['db_user'],
password=config['Database']['db_password'],
host=config['Database']['db_hostname'],
port=int(config['Database']['db_port']),
authentication_source=config['Database']['db_authentication'],
connect=False)
Project.drop_collection()
IssueSystem.drop_collection()
Issue.drop_collection()
IssueComment.drop_collection()
Event.drop_collection()
self.project_id = Project(name='Bla').save().id
self.issues_system_id = IssueSystem(
project_id=self.project_id,
url="https://issues.apache.org/search?jql=project=BLA",
last_updated=datetime.datetime.now()).save().id
self.conf = ConfigMock(
None, None, None, None, None, None, 'Bla',
'https://issues.apache.org/search?jql=project=BLA', 'jira', None,
None, None, None, None, None, 'DEBUG', '123')
def test_store_events(self, get_user_mock):
user1_obj = jira.resources.User(options=None,
session=None,
raw=self.user1)
user2_obj = jira.resources.User(options=None,
session=None,
raw=self.user2)
get_user_mock.side_effect = [user1_obj, user2_obj]
new_jira_backend = JiraBackend(self.conf, self.issues_system_id,
self.project_id)
issue = jira.resources.Issue(options=None,
session=None,
raw=self.issue_drill_138)
mongo_issue = Issue(external_id="TEST",
issue_system_id=self.issues_system_id).save()
new_jira_backend._store_events(issue, mongo_issue.id)
stored_events = Event.objects(issue_id=mongo_issue.id).all()
self.assertEqual(24, len(stored_events))
jacques_user = People.objects(email="*****@*****.**",
username="******").get()
timothy_user = People.objects(email="*****@*****.**",
username="******").get()
cmerrick_user = People.objects(email="*****@*****.**",
username="******").get()
# Check each event
# Custom event
event = stored_events[0]
self.assertEqual(event.status, "Target Version/s")
self.assertEqual(event.old_value, None)
self.assertEqual(event.new_value, "0.1")
self.assertEqual(event.created_at,
datetime.datetime(2016, 8, 13, 17, 29, 13, 718000))
self.assertEqual(event.author_id, jacques_user.id)
# Status
event = stored_events[1]
self.assertEqual(event.status, "status")
self.assertEqual(event.old_value, "Reopened")
self.assertEqual(event.new_value, "Resolved")
self.assertEqual(event.created_at,
datetime.datetime(2016, 8, 13, 17, 29, 13, 718000))
self.assertEqual(event.author_id, jacques_user.id)
# Resolution
event = stored_events[2]
self.assertEqual(event.status, "resolution")
self.assertEqual(event.old_value, "Test")
self.assertEqual(event.new_value, "Fixed")
self.assertEqual(event.created_at,
datetime.datetime(2016, 8, 13, 17, 29, 13, 718000))
self.assertEqual(event.author_id, jacques_user.id)
# Attachment
event = stored_events[3]
self.assertEqual(event.status, "Attachment")
self.assertEqual(event.old_value, None)
self.assertEqual(event.new_value, "testfails_ZOOKEEPER-136.patch")
self.assertEqual(event.created_at,
datetime.datetime(2016, 8, 13, 17, 29, 13, 718000))
self.assertEqual(event.author_id, jacques_user.id)
event = stored_events[4]
self.assertEqual(event.status, "Attachment")
self.assertEqual(event.old_value, "testfails_ZOOKEEPER-137.patch")
self.assertEqual(event.new_value, None)
self.assertEqual(event.created_at,
datetime.datetime(2016, 8, 13, 17, 29, 13, 718000))
self.assertEqual(event.author_id, jacques_user.id)
# Link
related_issue_1 = Issue.objects(external_id="ZOOKEEPER-232").get()
related_issue_2 = Issue.objects(external_id="ZOOKEEPER-231").get()
event = stored_events[5]
self.assertEqual(event.status, "issue_links")
self.assertEqual(event.old_value, None)
self.assertEqual(
event.new_value, {
'effect': 'relates to',
'issue_id': related_issue_1.id,
'type': 'Reference'
})
self.assertEqual(event.created_at,
datetime.datetime(2016, 8, 13, 17, 29, 13, 718000))
self.assertEqual(event.author_id, jacques_user.id)
event = stored_events[6]
self.assertEqual(event.status, "issue_links")
self.assertEqual(
event.old_value, {
'effect': 'relates to',
'issue_id': related_issue_2.id,
'type': 'Reference'
})
self.assertEqual(event.new_value, None)
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# assignee
assignee_old = People.objects(username="******").get()
assignee_new = People.objects(username="******").get()
event = stored_events[7]
self.assertEqual(event.status, "assignee_id")
self.assertEqual(event.old_value, assignee_old.id)
self.assertEqual(event.new_value, assignee_new.id)
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# fix version
event = stored_events[8]
self.assertEqual(event.status, "fix_versions")
self.assertEqual(event.old_value, None)
self.assertEqual(event.new_value, "3.6.0")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
event = stored_events[9]
self.assertEqual(event.status, "fix_versions")
self.assertEqual(event.old_value, "3.5.0")
self.assertEqual(event.new_value, None)
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# priority
event = stored_events[10]
self.assertEqual(event.status, "priority")
self.assertEqual(event.old_value, "Major")
self.assertEqual(event.new_value, "Blocker")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# issue type
event = stored_events[11]
self.assertEqual(event.status, "issue_type")
self.assertEqual(event.old_value, "New Feature")
self.assertEqual(event.new_value, "Improvement")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# affects version
event = stored_events[12]
self.assertEqual(event.status, "affects_versions")
self.assertEqual(event.old_value, None)
self.assertEqual(event.new_value, "3.0.0")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
event = stored_events[13]
self.assertEqual(event.status, "affects_versions")
self.assertEqual(event.old_value, "3.0.1")
self.assertEqual(event.new_value, None)
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# title
event = stored_events[14]
self.assertEqual(event.status, "title")
self.assertEqual(event.old_value, "Dump of ZooKeeper SVN repository")
self.assertEqual(event.new_value,
"Initial ZooKeeper code contribution from Yahoo!")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# desc
event = stored_events[15]
self.assertEqual(event.status, "desc")
self.assertEqual(
event.old_value,
"There are a couple of cases of member variables that need to be marked "
"volatile or surrounded in a synchronization block. A couple of examples "
"are:\n\n* QuorumPeer state should be synchronous\n* currentVote in "
"QuorumPeer is marked volatile, but when it's members are often accessed "
"individually as if they were in an atomic unit. Such code should be changed"
" to get a reference to the currentVote and they access members through that"
" reference.\n")
self.assertEqual(
event.new_value,
"There are a couple of cases of member variables that need to be marked "
"volatile or surrounded in a synchronization block. A couple of examples "
"are:\n\n* QuorumPeer state should be synchronous\n* currentVote in "
"QuorumPeer is marked volatile, but when it's members are often accessed "
"individually as if they were in an atomic unit. Such code should be "
"changed to get a reference to the currentVote and they access members "
"through that reference.\n* It looks like logicalClock in FastLeaderElection"
" should be volatile. It should either be fixed or commented to explain why "
"it doesn't need to be.")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# labels
event = stored_events[16]
self.assertEqual(event.status, "labels")
self.assertEqual(event.old_value, "newbie")
self.assertEqual(event.new_value, "docuentation newbie")
self.assertEqual(event.created_at,
datetime.datetime(2015, 2, 24, 17, 0, 58, 268000))
self.assertEqual(event.author_id, timothy_user.id)
# parent issue id
event = stored_events[17]
self.assertEqual(event.status, "parent_issue_id")
self.assertEqual(event.old_value, related_issue_1.id)
self.assertEqual(event.new_value, related_issue_2.id)
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
# environment
event = stored_events[18]
self.assertEqual(event.status, "environment")
self.assertEqual(
event.old_value,
"Sparc Solaris 10\r\nJava 6u17 64 bits\r\n5 nodes ensemble")
self.assertEqual(
event.new_value,
"Sparc Solaris 10 and 11\r\nJava 6u17 64 bits\r\n5 nodes ensemble")
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
# Custom labels
event = stored_events[19]
self.assertEqual(event.status, "labels")
self.assertEqual(event.old_value, "")
self.assertEqual(event.new_value, "zookeeper")
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
event = stored_events[20]
self.assertEqual(event.status, "labels")
self.assertEqual(event.old_value, "zookeeper1")
self.assertEqual(event.new_value, "")
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
# original_time_estimate
event = stored_events[21]
self.assertEqual(event.status, "original_time_estimate")
self.assertEqual(event.old_value, 480)
self.assertEqual(event.new_value, 28800)
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
# component
event = stored_events[22]
self.assertEqual(event.status, "components")
self.assertEqual(event.old_value, None)
self.assertEqual(event.new_value, "c client")
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
event = stored_events[23]
self.assertEqual(event.status, "components")
self.assertEqual(event.old_value, "python client")
self.assertEqual(event.new_value, None)
self.assertEqual(event.created_at,
datetime.datetime(2014, 2, 24, 17, 43, 46, 87000))
self.assertEqual(event.author_id, cmerrick_user.id)
def _process_event(self, unique_event_id, bz_event, mongo_issue,
change_date, author_id):
"""
Processes the event. During the event processing the Issue is set back to its original state
before the event occured.
:param unique_event_id: unique identifier of the event
:param bz_event: event that was received from the bugzilla API
:param mongo_issue: issue that is/should be stored in the mongodb
:param change_date: date when the event was created
:param author_id: :class:`bson.objectid.ObjectId` of the author of the event
"""
try:
mongo_event = Event.objects(external_id=unique_event_id,
issue_id=mongo_issue.id).get()
return mongo_event, False
except DoesNotExist:
mongo_event = Event(external_id=unique_event_id,
issue_id=mongo_issue.id,
created_at=change_date,
author_id=author_id)
# We need to map back the status from the bz terminology to ours. Special: The assigned_to must be mapped to
# assigned_to_detail beforehand, as we are using this for the issue parsing
if bz_event['field_name'] == 'assigned_to':
bz_at_name = 'assigned_to_detail'
else:
bz_at_name = bz_event['field_name']
try:
mongo_event.status = self.at_mapping[bz_at_name]
except KeyError:
logger.warning('Mapping for attribute %s not found.' % bz_at_name)
mongo_event.status = bz_at_name
if mongo_event.status == 'assignee_id':
if bz_event['added'] is not None and bz_event['added']:
people_id = self._get_people(bz_event['added'])
mongo_event.new_value = people_id
if bz_event['removed'] is not None and bz_event['removed']:
people_id = self._get_people(bz_event['removed'])
mongo_event.old_value = people_id
elif bz_event['field_name'] == 'depends_on':
if bz_event['added'] is not None and bz_event['added']:
issue_id = self._get_issue_id_by_system_id(bz_event['added'])
mongo_event.new_value = {
'issue_id': issue_id,
'type': 'Dependent',
'effect': 'depends on'
}
if bz_event['removed'] is not None and bz_event['removed']:
issue_id = self._get_issue_id_by_system_id(bz_event['removed'])
mongo_event.old_value = {
'issue_id': issue_id,
'type': 'Dependent',
'effect': 'depends on'
}
elif bz_event['field_name'] == 'blocks':
if bz_event['added'] is not None and bz_event['added']:
issue_id = self._get_issue_id_by_system_id(bz_event['added'])
mongo_event.new_value = {
'issue_id': issue_id,
'type': 'Blocker',
'effect': 'blocks'
}
if bz_event['removed'] is not None and bz_event['removed']:
issue_id = self._get_issue_id_by_system_id(bz_event['removed'])
mongo_event.old_value = {
'issue_id': issue_id,
'type': 'Blocker',
'effect': 'blocks'
}
else:
if bz_event['added'] is not None and bz_event['added']:
mongo_event.new_value = bz_event['added']
if bz_event['removed'] is not None and bz_event['removed']:
mongo_event.old_value = bz_event['removed']
return mongo_event, True