def setUp(self):
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue1.json", 'r', encoding='utf-8') as \
issue_1_file:
self.issue_1 = json.load(issue_1_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue95.json", 'r', encoding='utf-8') as \
issue_95_file:
self.issue_95 = json.load(issue_95_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue95_comments.json", 'r', encoding='utf-8') as \
issue_95_comments_file:
self.issue_95_comments = json.load(issue_95_comments_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/issue95_history.json", 'r', encoding='utf-8') as \
issue_95_history_file:
self.issue_95_history = json.load(issue_95_history_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/conor_apache_org_user.json", 'r', encoding='utf-8') as \
conor_user_file:
self.conor_user = json.load(conor_user_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/dev_tomcat_apache_org_user.json", 'r', encoding='utf-8') as \
dev_tomcat_file:
self.dev_tomcat_file = json.load(dev_tomcat_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/bugzilla/craig_mcclanahan_user.json", 'r', encoding='utf-8') as \
craig_user_file:
self.craig_user = json.load(craig_user_file)
# Create testconfig
config = configparser.ConfigParser()
config.read(
os.path.dirname(os.path.realpath(__file__)) +
"/data/used_test_config.cfg")
# Setting up database with data that is normally put into it via vcs program
connect(config['Database']['db_database'],
username=config['Database']['db_user'],
password=config['Database']['db_password'],
host=config['Database']['db_hostname'],
port=int(config['Database']['db_port']),
authentication_source=config['Database']['db_authentication'],
connect=False)
Project.drop_collection()
IssueSystem.drop_collection()
Issue.drop_collection()
IssueComment.drop_collection()
Event.drop_collection()
self.project_id = Project(name='Bla').save().id
self.issues_system_id = IssueSystem(
project_id=self.project_id,
url="https://issues.apache.org/search?jql=project=BLA",
last_updated=datetime.datetime.now()).save().id
self.conf = ConfigMock(None, None, None, None, None, None, 'Bla',
'Nonsense?product=Blub', 'bugzilla', None, None,
None, None, None, None, 'DEBUG', '123')
def isfeatureadd(issue):
its = IssueSystem.objects(id=issue.issue_system_id).get()
if 'jira' in its.url:
return _is_jira_featureadd(issue)
else:
log.error('unknown ITS type for ITS url %s for feature add labels' %
its.url)
return False
def setUp(self):
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/people.json", 'r', encoding='utf-8') as people_file:
self.person = json.load(people_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6131.json", 'r', encoding='utf-8') as issues_file:
self.issue_6131 = json.load(issues_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6131_events.json", 'r', encoding='utf-8') as event_file:
self.events_issue_6131 = json.load(event_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6131_comments.json", 'r', encoding='utf-8') as cmt_file:
self.comments_issue_6131 = json.load(cmt_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6050.json", 'r', encoding='utf-8') as issues_file:
self.issue_6050 = json.load(issues_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6050_events.json", 'r', encoding='utf-8') as event_file:
self.events_issue_6050 = json.load(event_file)
with open(os.path.dirname(os.path.realpath(__file__)) + "/data/github/issue_6050_comments.json", 'r', encoding='utf-8') as cmt_file:
self.comments_issue_6050 = json.load(cmt_file)
# Create testconfig
config = configparser.ConfigParser()
config.read(os.path.dirname(os.path.realpath(__file__)) + "/data/used_test_config.cfg")
# Setting up database with data that is normally put into it via vcs program
connect(config['Database']['db_database'], username=config['Database']['db_user'],
password=config['Database']['db_password'], host=config['Database']['db_hostname'],
port=int(config['Database']['db_port']),
authentication_source=config['Database']['db_authentication'],
connect=False)
Project.drop_collection()
IssueSystem.drop_collection()
Issue.drop_collection()
IssueComment.drop_collection()
Event.drop_collection()
self.project_id = Project(name='Composer').save().id
self.issues_system_id = IssueSystem(project_id=self.project_id, url="http://blub.de",
last_updated=datetime.datetime.now()).save().id
self.conf = ConfigMock(None, None, None, None, None, None, 'Ant', 'http://blub.de', 'github', None, None, None,
None, None, None, 'DEBUG', '123')
def isbugfix(issue):
its = IssueSystem.objects(id=issue.issue_system_id).get()
if 'jira' in its.url:
return _jira_isbugfix(issue)
elif 'bugzilla' in its.url:
return _bz_isbugfix(issue)
elif 'github' in its.url:
return _gh_isbugfix(issue)
else:
log.error('unknown ITS type for ITS url %s for bugfix labels' %
its.url)
return False
def start(self, cfg):
"""
Starts the collection process
:param cfg: holds all configuration parameters. Object of class :class:`~issueshark.config.Config`
"""
logger.setLevel(cfg.get_debug_level())
start_time = timeit.default_timer()
# Connect to mongodb
uri = create_mongodb_uri_string(cfg.user, cfg.password, cfg.host,
cfg.port, cfg.authentication_db,
cfg.ssl_enabled)
connect(cfg.database, host=uri)
# Get the project for which issue data is collected
try:
project_id = Project.objects(name=cfg.project_name).get().id
except DoesNotExist:
logger.error('Project %s not found!' % cfg.project_name)
sys.exit(1)
# Create issue system if not already there
try:
issue_system = IssueSystem.objects(url=cfg.tracking_url).get()
except DoesNotExist:
issue_system = IssueSystem(project_id=project_id,
url=cfg.tracking_url).save()
issue_system.last_updated = datetime.datetime.now()
issue_system.save()
# Find correct backend
backend = BaseBackend.find_fitting_backend(cfg, issue_system.id,
project_id)
logger.debug("Using backend: %s" % backend.identifier)
# Process the issues for the corresponding project_id
backend.process()
elapsed = timeit.default_timer() - start_time
logger.info("Execution time: %0.5f s" % elapsed)
def setUp(self):
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/drill_1_issue.json",
'r',
encoding='utf-8') as drill_1:
self.issue_drill_1 = json.load(drill_1)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/drill_138_issue.json",
'r',
encoding='utf-8') as drill_138:
self.issue_drill_138 = json.load(drill_138)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/drill_38_issue.json",
'r',
encoding='utf-8') as drill_38:
self.issue_drill_38 = json.load(drill_38)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/get_user1.json",
'r',
encoding='utf-8') as user1_file:
self.user1 = json.load(user1_file)
with open(os.path.dirname(os.path.realpath(__file__)) +
"/data/jira/get_user2.json",
'r',
encoding='utf-8') as user2_file:
self.user2 = json.load(user2_file)
# Create testconfig
config = configparser.ConfigParser()
config.read(
os.path.dirname(os.path.realpath(__file__)) +
"/data/used_test_config.cfg")
# Setting up database with data that is normally put into it via vcs program
connect(config['Database']['db_database'],
username=config['Database']['db_user'],
password=config['Database']['db_password'],
host=config['Database']['db_hostname'],
port=int(config['Database']['db_port']),
authentication_source=config['Database']['db_authentication'],
connect=False)
Project.drop_collection()
IssueSystem.drop_collection()
Issue.drop_collection()
IssueComment.drop_collection()
Event.drop_collection()
self.project_id = Project(name='Bla').save().id
self.issues_system_id = IssueSystem(
project_id=self.project_id,
url="https://issues.apache.org/search?jql=project=BLA",
last_updated=datetime.datetime.now()).save().id
self.conf = ConfigMock(
None, None, None, None, None, None, 'Bla',
'https://issues.apache.org/search?jql=project=BLA', 'jira', None,
None, None, None, None, None, 'DEBUG', '123')
def start(self, cfg):
"""
Executes the linkSHARK.
:param cfg: configuration object that is used
"""
self._log.setLevel(cfg.get_debug_level())
start_time = timeit.default_timer()
uri = create_mongodb_uri_string(cfg.user, cfg.password, cfg.host,
cfg.port, cfg.authentication_db,
cfg.ssl_enabled)
connect(cfg.database, host=uri)
# Get the id of the project for which the code entities shall be merged
try:
project_id = Project.objects(name=cfg.project_name).get().id
except DoesNotExist:
self._log.error('Project %s not found!' % cfg.project_name)
sys.exit(1)
vcs_system = VCSSystem.objects(project_id=project_id).get()
self._itss = []
self._log.info('found the following issue tracking systems:')
for its in IssueSystem.objects(project_id=project_id).order_by('url'):
self._log.info(its.url)
self._itss.append(its)
if len(cfg.correct_key) > 0:
correct_keys_per_its = cfg.correct_key.split(';')
if len(correct_keys_per_its) != len(self._itss):
self._log_critical(
'--correct-key must correct keys for all issue tracking systems if specified'
)
sys.exit(1)
for i, correct_key in enumerate(correct_keys_per_its):
self._correct_key[self._itss[i].url] = correct_key
if len(cfg.broken_keys) > 0:
broken_keys_per_its = cfg.broken_keys.split(';')
if len(broken_keys_per_its) != len(self._itss):
self._log_critical(
'--broken-keys must correct keys for all issue tracking systems if specified. If there are no keys to correct for one of the ITS just use the name of the correct key twice itself'
)
sys.exit(1)
for i, broken_keys in enumerate(broken_keys_per_its):
self._broken_keys[self._itss[i].url] = broken_keys.split(',')
self._log.info("Starting issue linking")
commit_count = Commit.objects(vcs_system_id=vcs_system.id).count()
issue_map = {}
for i, issue_system in enumerate(self._itss):
project_id_string = correct_keys_per_its[i]
for issue in Issue.objects(issue_system_id=issue_system.id):
if issue.external_id.startswith(project_id_string):
try:
issue_number = [
int(s) for s in issue.external_id.split('-')
if s.isdigit()
][0]
except IndexError:
self._log.error(
"index error because SZZ currently only support JIRA, may not link all issues correctly:",
issue.external_id)
continue
if issue_number not in issue_map:
issue_map[issue_number] = [issue]
else:
issue_map[issue_number].append(issue)
for i, commit in enumerate(
Commit.objects(vcs_system_id=vcs_system.id).only(
'id', 'revision_hash', 'vcs_system_id', 'message',
'author_id', 'committer_id')):
if i % 100 == 0:
self._log.info("%i/%i commits finished", i, commit_count)
issue_links = self._get_issue_links(commit)
if len(issue_links) > 0:
commit.linked_issue_ids = issue_links
commit.save()
szz_links = self._get_szz_issue_links(commit, issue_map)
if len(szz_links) > 0:
commit.szz_issue_ids = szz_links
commit.save()
elapsed = timeit.default_timer() - start_time
self._log.info("Execution time: %0.5f s" % elapsed)
# Fetch project id and version control system id for the 'kafka' project
# The only() decides the data that is actually retrieved from the MongoDB. Always restrict this to the field that you require!
projects = ['ant-ivy', 'archiva', 'calcite', 'cayenne', 'commons-bcel', 'commons-beanutils', 'commons-codec', 'commons-collections',
'commons-compress', 'commons-configuration', 'commons-dbcp', 'commons-digester', 'commons-io',
'commons-jcs', 'commons-jexl', 'commons-lang', 'commons-math', 'commons-net', 'commons-rdf', 'commons-scxml']
rows_list = []
for projectName in projects:
project = Project.objects(name=projectName).only('id').get()
#vcs_system = VCSSystem.objects(project_id=project.id).only('id','url').get()
#getting issue id from the project
issue_id = IssueSystem.objects(project_id=project.id).only('id','url').get()
###########Getting data ready############
for issue in Issue.objects(issue_system_id=issue_id.id).only('issue_type','desc','title','priority', 'status').timeout(False):
for row in issue:
dict1 = {}
dict1.update({'Id':issue_id.id})
dict1.update({'Description':issue.desc})
dict1.update({'Title':issue.title})
dict1.update({'Issue_Type':issue.issue_type})
rows_list.append(dict1)
rows_list
df = pd.DataFrame(rows_list)