def signal_handler(signal, frame):
# prevent double-print on debug
if not utils.debug:
print('\nShutting down Argus on SigInt.')
argus_debug('Shutting down Argus on SigInt.')
if utils.argus_log is not None:
utils.argus_log.close()
sys.exit(0)
def _translate_field(self, jira_issue: 'JiraIssue') -> str:
"""
For this issue, parse out the JiraProject it belongs to and translate our local field's readable text to
whatever cf* is on the project side
"""
jira_project = self._jira_connection.maybe_get_cached_jira_project(jira_issue.project_name)
if jira_project is None:
return 'None'
argus_debug('JiraFilter: Attempting to translate {} for jira_issue: {}'.format(
self.field, jira_issue.issue_key))
return jira_project.translate_custom_field(self.field)
def _internal_matching_operation(self, jira_issue: 'JiraIssue', to_match: List[str]) -> bool:
matches_one = False
matches_all = True
translated = self._translate_field(jira_issue)
in_issue = translated in jira_issue
value = 'Not found'
if in_issue:
value = jira_issue[translated]
argus_debug('Checking for translated field {} in issue: {}. Found: {}. Value: {}. Filter: {}'.format(
translated, jira_issue.issue_key, in_issue, value, self))
if translated in jira_issue:
argus_debug('Checking for {} in {}'.format(translated, jira_issue.issue_key))
for match in to_match:
argus_debug('Checking against match: {}'.format(match))
if match in jira_issue[translated]:
argus_debug(' FOUND MATCH')
matches_one = True
else:
matches_all = False
if self.query_type() == 'OR':
return matches_one
return matches_one and matches_all
def since(source, delta):
# type: (datetime, str) -> datetime
"""
Accepts string in format '-123d 3w 5m', negative optional on each, can take multiple space delim options
:return: datetime object representing deltasd interval from datetime.now()
"""
day_delta = _extract_time('d', delta)
week_delta = _extract_time('w', delta)
month_delta = _extract_time('m', delta)
year_delta = _extract_time('y', delta)
utils.argus_debug('since input source: {}. delta: [{}]. days:{} weeks:{} months:{} years:{}'.format(
source, delta, day_delta, week_delta, month_delta, year_delta
))
return source + relativedelta(days=day_delta, weeks=week_delta, months=month_delta, years=year_delta)
def populate_jira_issues(self, jira_connection: 'JiraConnection',
issues: List[List['JiraIssue']]) -> None:
count_added = 0
for list_of_issues in issues:
for jira_issue in list_of_issues:
for member in list(self._team_members.values()):
# Can't short-circuit here since one member may be assignee and another reviewer
if member.add_if_owns(jira_connection, jira_issue):
count_added += 1
utils.argus_debug(
'Team: {}. JiraConnection: {}. Count added: {}'.format(
self.name, jira_connection.connection_name, count_added))
for member in list(self._team_members.values()):
utils.argus_debug(
'At end of add_owned_issues for team: {}. Member: {}'.format(
self.name, member))
def add_field_translations_from_file(self):
"""
Pulls custom translations from conf/custom_params.cfg and initializes this JiraProject with them if they are
not otherwise defined
"""
if not os.path.exists('conf/custom_params.cfg'):
return
else:
config_parser = configparser.RawConfigParser()
config_parser.read('conf/custom_params.cfg')
# Current limitation: one hard-coded set of config per project name. i.e. one URL
if config_parser.has_section(self.project_name):
url = config_parser.get(self.project_name, 'url').rstrip('/')
if url != self._url:
msg = 'WARNING! Found project {} but url mismatched (project: {} and file: {}). Not loading translations.'
print(msg.format(self.project_name, self._url, url))
return
field_names = config_parser.get(self.project_name,
'custom_fields').split(',')
changed = False
for field in field_names:
if field not in self._custom_fields:
for cf in list(self._custom_fields.keys()):
print(' Known: {}'.format(cf))
changed = True
translated_value = config_parser.get(
self.project_name, field)
print(
'Adding missing custom field translation from conf/custom_params.cfg for '
'project: {} field: {} value: {}'.format(
self.project_name, field, translated_value))
self._custom_fields[field] = translated_value
if changed:
print(
'Migrated custom params from conf/custom_params.cfg into project: {}. Saving config.'
.format(self.project_name))
self.save_config()
else:
argus_debug(
'No changes from custom_params necessary for {}'.format(
self.project_name))
def from_file(cls) -> 'TeamManager':
print('Loading Team config from file')
config_parser = configparser.RawConfigParser()
try:
result = TeamManager()
if not os.path.exists(os.path.join(conf_dir, 'teams.cfg')):
argus_debug('Did not find any existing conf/teams.cfg file. Empty TeamManager.')
return result
config_parser.read(os.path.join(conf_dir, 'teams.cfg'))
# Add teams
if config_parser.has_section('manager'):
team_roots = config_parser.get('manager', 'team_names').split(',')
for team_root in team_roots:
# Skip trailing ,
if team_root == '':
continue
name, jira_connection_name = team_root.split(':')
result._teams[name] = Team(name, jira_connection_name)
argus_debug('TeamManager.init: Adding team: {} from config'.format(name))
# Add MemberIssuesByStatus
for member_root_name in config_parser.sections():
# TODO: Consider removing these two manualy bypasses. Kind of hacky to assume everything in config is member root.
if member_root_name == 'manager' or member_root_name == 'organizations':
continue
new_member = MemberIssuesByStatus.from_file(member_root_name, config_parser)
team = result.get_team_by_name(new_member.primary_team)
if team is None:
raise ValueError('Failed to find a constructed team with name: {}'.format(new_member.primary_team))
team.add_existing_member(new_member)
argus_debug('TeamManager init: Adding team member: {}'.format(new_member.full_name))
# Init Orgs
if config_parser.has_section('organizations'):
for org_name in config_parser.get('organizations', 'org_names').split(','):
new_org = set()
for team_name in config_parser.get('organizations', org_name).split(','):
new_org.add(team_name)
result._organizations[org_name] = new_org
return result
except (AttributeError, ValueError, IOError) as e:
print('Exception during creation of TeamManager. Config file name: {}. Exception stack follows:'.format(os.path.join(conf_dir, 'teams.cfg')))
traceback.print_exc()
raise e
def __init__(self, team_manager):
"""
Recreates any JiraConnections and JiraViews based on saved data in conf/jira.cfg
"""
# Holds connected Jira objects to be queried by JiraViews
self._jira_connections = {} # type: Dict[str, JiraConnection]
# JiraViews, caching filters for different ways to view Jira Data. Implicit 1:1 JiraConnection to JiraView
self.jira_views = {} # type: Dict[str, JiraView]
self.jira_dashboards = {} # type: Dict[str, JiraDashboard]
# Used during JiraDependency resolution to notify user of missing JiraProjects
self.missing_project_counts = {} # type: Dict[str, int]
self._display_filter = DisplayFilter.default()
if os.path.exists(jira_conf_file):
config_parser = configparser.RawConfigParser()
config_parser.read(jira_conf_file)
connection_names = []
if config_parser.has_section(
'JiraManager') and config_parser.has_option(
'JiraManager', 'Connections'):
connection_names = config_parser.get('JiraManager',
'Connections').split(',')
# JiraConnections are the root of our container hierarchy
for connection_name in connection_names:
if connection_name == '':
pass
try:
jira_connection = JiraConnection.from_file(connection_name)
# If we had an error on init we obviously cannot add this
if jira_connection is None:
continue
self._jira_connections[
jira_connection.connection_name] = jira_connection
except ConfigError as ce:
print('ConfigError with project {}: {}'.format(
connection_name, ce))
# Construct JiraViews so they can be used during JiraDashboard creation.
view_names = []
if config_parser.has_option('JiraManager', 'Views'):
view_names = config_parser.get('JiraManager',
'Views').split(',')
for name in view_names:
try:
jv = JiraView.from_file(self, name, team_manager)
self.jira_views[jv.name] = jv
except ConfigError as ce:
print('ConfigError with jira view {}: {}'.format(name, ce))
if config_parser.has_section('Dashboards'):
for dash in config_parser.options('Dashboards'):
dash_views = {}
for view in view_names:
if view not in self.jira_views:
print(
'Found dashboard {} with invalid view: {}. Skipping init: manually remove from config.'
.format(dash, view))
break
dash_views[view] = self.jira_views[view]
self.jira_dashboards[dash] = JiraDashboard(
dash, dash_views)
if len(self._jira_connections) == 0:
print_separator(30)
print(
'No JIRA Connections found. Prompting to add first connection.'
)
self.add_connection()
# Initialize JiraProjects from locally cached files
for file_name in os.listdir(jira_project_dir):
full_path = os.path.join(jira_project_dir, file_name)
print(
'Processing locally cached JiraProject: {}'.format(full_path))
# Init based on matching the name of this connection and .cfg
print_separator(30)
try:
new_jira_project = JiraProject.from_file(full_path, self)
if new_jira_project is None:
print('Error initializing from {}. Skipping'.format(
full_path))
break
if new_jira_project.jira_connection is None:
add = get_input(
'Did not find JiraConnection for JiraProject: {}. Would you like to add one now? (y/n)'
)
if add == 'y':
new_jira_connection = self.add_connection(
'Name the connection (reference url: {}):'.format(
new_jira_project.url))
new_jira_connection.save_config()
new_jira_project.jira_connection = new_jira_connection
else:
print(
'Did not add JiraConnection, so cannot link and use JiraProject.'
)
continue
print('Updating with new data from JIRA instance')
new_jira_project.refresh()
new_jira_project.jira_connection.add_and_link_jira_project(
new_jira_project)
except (configparser.NoSectionError, ConfigError) as e:
print(
'WARNING! Encountered error initializing JiraProject from file {}: {}'
.format(full_path, e))
print(
'This JiraProject will not be initialized. Remove it manually from disk in conf/jira/projects and data/jira/'
)
if os.path.exists('conf/custom_params.cfg'):
config_parser = configparser.RawConfigParser()
config_parser.read('conf/custom_params.cfg')
custom_projects = config_parser.get('CUSTOM_PROJECTS',
'project_names').split(',')
for project_name in custom_projects:
argus_debug(
'Processing immutable config for custom project: {}'.
format(project_name))
# Find the JiraConnection w/matching URL, if any
url = config_parser.get(project_name, 'url').rstrip('/')
jira_project = self.maybe_get_cached_jira_project(
url, project_name)
if jira_project is not None:
# Don't need to cache since already done on ctor for JiraProject
argus_debug('Project already initialized. Skipping.')
continue
# Didn't find the JiraProject, so we need to build one, cache, and link.
custom_fields = {}
field_names = config_parser.get(project_name,
'custom_fields').split(',')
for field in field_names:
custom_fields[field] = config_parser.get(
project_name, field)
parent_jira_connection = None
for jira_connection in list(self._jira_connections.values()):
if jira_connection.url == url:
parent_jira_connection = jira_connection
break
# Create a JiraConnection for this JiraProject if we do not yet have one
if parent_jira_connection is None:
print(
'WARNING! Did not find JiraConnection for project: {}, attempting to match url: {}'
.format(project_name, url))
print('Known JiraConnections and their urls:')
for jira_connection in list(
self._jira_connections.values()):
print(' {}: {}'.format(
jira_connection.connection_name,
jira_connection.url))
if is_yes('Would you like to add one now?'):
parent_jira_connection = self.add_connection(
'Name the connection (reference url: {}):'.format(
url))
else:
print(
'JiraProject data and config will not be added nor cached. Either add it manually or restart Argus and reply y'
)
break
new_jira_project = JiraProject(parent_jira_connection,
project_name, url,
custom_fields)
new_jira_project.refresh()
parent_jira_connection.add_and_link_jira_project(
new_jira_project)
print('Resolving dependencies between JiraIssues')
self._resolve_issue_dependencies()
print('JiraManager initialization complete.')
def get_issues(self, string_matches=None):
# type: (List[str]) -> Dict[str, JiraIssue]
"""
Applies nested JiraFilters to all associated cached JiraProjects for the contained JiraConnection
:param string_matches: substring(s) to match against JiraIssue fields for further refining of a search
:return: {} of key -> JiraIssue that match JiraFilters and input regexes
"""
if string_matches is None:
string_matches = []
source_issues = self.jira_connection.cached_jira_issues
matching_issues = {}
excluded_count = 0
for issue_list in source_issues:
for jira_issue in issue_list:
matched = False
has_or = False
matched_or = False
if utils.debug:
print_separator(30)
argus_debug(
'Matching against JiraIssue with key: {key}, assignee: {assignee}, rev: {rev}, rev2: {rev2}, res: {res}'
.format(
key=jira_issue.issue_key,
assignee=jira_issue['assignee'],
rev=jira_issue.get_value(self.jira_connection,
'reviewer'),
rev2=jira_issue.get_value(self.jira_connection,
'reviewer2'),
res=jira_issue.get_value(self.jira_connection,
'resolution')))
for jira_filter in list(self._jira_filters.values()):
argus_debug(
'Processing filter: {}'.format(jira_filter))
excluded = False
argus_debug('Checking jira_filter match for issue: {}'.format(
jira_issue.issue_key))
for jira_filter in list(self._jira_filters.values()):
argus_debug('Processing filter: {}'.format(jira_filter))
# if we have an OR filter in the JiraFilter, we need to match at least one to be valid
if jira_filter.query_type() == 'OR':
has_or = True
if not jira_issue.matches_any(self.jira_connection,
string_matches):
argus_debug(
' Skipping {}. Didn\'t match regexes: {}'.format(
jira_filter.extract_value(jira_issue),
','.join(string_matches)))
excluded_count += 1
break
if jira_filter.includes_jira_issue(jira_issue):
argus_debug(' Matched: {} with value: {}'.format(
jira_filter,
jira_filter.extract_value(jira_issue)))
matched = True
if jira_filter.query_type() == 'OR':
matched_or = True
elif jira_filter.excludes_jira_issue(jira_issue):
argus_debug(' Excluded by: {} with value: {}'.format(
jira_filter,
jira_filter.extract_value(jira_issue)))
matched = True
excluded = True
break
# Didn't match and is required, we exclude this JiraIssue
elif jira_filter.query_type() == 'AND':
argus_debug(
' Didn\'t match: {} with value: {} and was AND. Excluding.'
.format(jira_filter,
jira_filter.extract_value(jira_issue)))
excluded = True
# Didn't match and was OR, don't flag anything
else:
argus_debug(
' Didn\'t match: {} with value and was OR. Doing nothing: {}'
.format(jira_filter,
jira_filter.extract_value(jira_issue)))
# Cannot short-circuit on match since exclusion beats inclusion and we have to keep checking, but can
# on exclusion bool
if excluded:
excluded_count += 1
break
argus_debug(' key: {} matched: {}. excluded: {}'.format(
jira_issue.issue_key, matched, excluded))
if not excluded:
if has_or and not matched_or:
argus_debug(
' has_or on filter, did not match on or field. Excluding.'
)
elif matched:
matching_issues[jira_issue.issue_key] = jira_issue
print(
'Returning total of {} JiraIssues matching JiraView {}. Excluded count: {}'
.format(len(list(matching_issues.keys())), self.name,
excluded_count))
return matching_issues
def __init__(self, jira_manager: JiraManager, team_manager: TeamManager,
options: Dict[str, str]) -> None:
self.jira_manager = jira_manager
self.team_manager = team_manager
# TODO: Clean up the coupling with main_menu.
self.jenkins_manager = JenkinsManager(self)
if 'triage_csv' in options:
jira_connections = {}
for jira_connection in self.jira_manager.jira_connections():
argus_debug('Init connection: {}'.format(
jira_connection.connection_name))
jira_connections[
jira_connection.connection_name] = jira_connection
triage_update = TriageUpdate(
jira_connections,
self.jira_manager.get_all_cached_jira_projects())
triage_out = options[
'triage_out'] if 'triage_out' in options else None
triage_update.process(options['triage_csv'], triage_out)
if 'dashboard' in options:
user_key = options['dashboard']
dash_keys = list(self.jira_manager.jira_dashboards.keys())
if user_key in dash_keys:
self.jira_manager.jira_dashboards[user_key].display_dashboard(
self.jira_manager, self.jira_manager.jira_views)
else:
print('Oops... Error with dashboard name {}'.format(user_key))
print('Possible dashboard names : {}'.format(
','.join(dash_keys)))
print('Starting Argus normally...')
if 'verbose' in options:
utils.debug = True
utils.argus_log = open('argus.log', 'w')
self.main_menu = [
MenuOption('d',
'Dashboards',
self.go_to_dashboards_menu,
pause=False),
MenuOption('v',
'Jira Views',
self.go_to_jira_views_menu,
pause=False),
MenuOption('p',
'JiraProject Queries',
self.go_to_projects_menu,
pause=False),
MenuOption.print_blank_line(),
MenuOption('t',
'Run a Team-Based Report',
self._run_team_report,
pause=False),
MenuOption('u',
'Run an org-based Report',
self._run_org_report,
pause=False),
MenuOption('e',
'View Escalations',
self.jira_manager.display_escalations,
pause=False),
MenuOption.print_blank_line(),
MenuOption('r',
'Generate a Pre-Determined Report',
self.go_to_reports_menu,
pause=False),
MenuOption('m',
'Team Management',
self.go_to_teams_menu,
pause=False),
MenuOption('c',
'Jira Connections',
self.go_to_jira_connections_menu,
pause=False),
MenuOption.print_blank_line(),
MenuOption('j',
'Jenkins Menu',
self.go_to_jenkins_menu,
pause=False),
MenuOption.print_blank_line(),
MenuOption('o',
'Change Options',
self.go_to_options_menu,
pause=False),
MenuOption.print_blank_line(),
MenuOption('h', 'Help', self._display_readme, pause=False),
MenuOption.quit_program()
]
if Config.Experiment is True:
self.main_menu.append(MenuOption.print_blank_line())
self.main_menu.append(
MenuOption('x',
'Debug',
self.jira_manager.run_debug,
pause=False))
self.dashboards_menu = [
MenuOption('l', 'List all available dashboards',
self.jira_manager.list_dashboards),
MenuOption('d', 'Display a dashboard\'s results',
self.jira_manager.display_dashboard),
MenuOption('c', 'Create a dashboard',
self.jira_manager.add_dashboard),
MenuOption('e', 'Edit a dashboard',
self.jira_manager.edit_dashboard),
MenuOption('r', 'Remove a dashboard',
self.jira_manager.remove_dashboard),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.jira_views_menu = [
MenuOption('l', 'List all defined JiraViews',
self.jira_manager.list_all_jira_views),
MenuOption('d', 'Display a JiraView\'s results',
self.jira_manager.display_view),
MenuOption('a', 'Add a JiraView', self._add_view),
MenuOption('e', 'Edit a JiraView', self._edit_view),
MenuOption('r', 'Remove a JiraView',
self.jira_manager.remove_view),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.reports_menu = [
MenuOption(
'f',
'FixVersion report (release). Query all tickets with a specified FixVersion',
self.jira_manager.report_fix_version),
MenuOption('s',
'Add a single-user multi-JIRA open ticket dashboard',
self._add_multi_jira_dashboard),
MenuOption('l', 'Add a label-based cross-cutting view',
self.jira_manager.add_label_view),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.team_menu = [
MenuOption('l', 'List all defined Teams',
self.team_manager.list_teams),
MenuOption('a', 'Add a new team', self._add_team),
MenuOption('e', 'Edit an existing team', self._edit_team),
MenuOption('r', 'Remove a team', self.team_manager.remove_team),
MenuOption(
'x',
'Link a team member to two accounts across JiraConnections',
self.add_linked_member),
MenuOption('d', 'Delete a cross-Jira link',
self.team_manager.remove_linked_member),
MenuOption('o', 'Add an organization',
self.team_manager.add_organization),
MenuOption('p', 'Remove an organization',
self.team_manager.remove_organization),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.jira_connections_menu = [
MenuOption('a', 'Add a JIRA connection',
self.jira_manager.add_connection),
MenuOption('r', 'Remove a JIRA connection and all related views',
self.jira_manager.remove_connection),
MenuOption(
'c',
'Cache offline ticket data for a JiraProject on a connection',
self.jira_manager.cache_new_jira_project_data),
MenuOption(
'd',
'Delete offline cached ticket data for a JiraProject on a connection',
self.jira_manager.delete_cached_jira_project),
MenuOption('l', 'List all configured Jiraconnections',
self.jira_manager.list_jira_connections),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.jenkins_menu = [
MenuOption('r',
'Reports Manager',
self.go_to_jenkins_reports_manager_menu,
pause=False),
MenuOption('c',
'Connections Manager',
self.go_to_jenkins_connections_manager_menu,
pause=False),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.jenkins_reports_manager_menu = [
MenuOption('o',
'Open custom report',
self.jenkins_manager.select_active_report,
pause=False),
MenuOption('a',
'Add a custom report',
self.jenkins_manager.add_custom_report,
pause=False),
MenuOption('r',
'Remove a custom report',
self.jenkins_manager.remove_custom_report,
pause=False),
MenuOption('l', 'List custom reports',
self.jenkins_manager.list_custom_reports),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_jenkins_menu)
]
self.jenkins_report_menu = [
MenuOption('v', 'View report',
self.jenkins_manager.view_custom_report),
MenuOption('a',
'Add a job',
self.jenkins_manager.add_custom_report_job,
pause=False),
MenuOption('r',
'Remove a job',
self.jenkins_manager.remove_custom_report_job,
pause=False),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(
self.go_to_jenkins_reports_manager_menu)
]
self.jenkins_connections_manager_menu = [
MenuOption('o',
'Open connection',
self.jenkins_manager.select_active_connection,
pause=False),
MenuOption('a',
'Add a connection',
self.jenkins_manager.add_connection,
pause=False),
MenuOption('r',
'Remove a connection',
self.jenkins_manager.remove_connection,
pause=False),
MenuOption('l', 'List connections',
self.jenkins_manager.list_connections),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_jenkins_menu)
]
self.jenkins_connection_menu = [
MenuOption('v',
'View cached jobs',
self.jenkins_manager.view_cached_jobs,
pause=False),
MenuOption('d',
'Download jobs to cache',
self.jenkins_manager.download_jobs,
pause=False),
MenuOption.print_blank_line(),
MenuOption('l', 'List saved views',
self.jenkins_manager.list_views),
MenuOption('a',
'Add a view',
self.jenkins_manager.add_view,
pause=False),
MenuOption('r',
'Remove a view',
self.jenkins_manager.remove_view,
pause=False),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(
self.go_to_jenkins_connections_manager_menu)
]
self.options_menu = [
MenuOption('p', 'Change Argus password', self._change_password),
MenuOption('b', 'Change browser', self._change_browser),
MenuOption('v', 'Toggle Verbose/Debug', self._change_debug),
MenuOption('d', 'Toggle Display dependencies',
self._change_show_dependencies),
MenuOption('o', 'Toggle show open dependencies only',
self._change_dependency_type),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.projects_menu = [
MenuOption('l',
'List locally cached projects',
self.jira_manager.list_projects,
pause=True),
MenuOption('s',
'Search locally cached JiraIssues for a string',
self.jira_manager.search_projects,
pause=False),
MenuOption('a',
'Add new JiraProject offline cache',
self.jira_manager.cache_new_jira_project_data,
pause=True),
MenuOption(
'd',
'Delete offline cached ticket data for a JiraProject on a connection',
self.jira_manager.delete_cached_jira_project),
MenuOption('u',
'Update all locally cached project JIRA data',
self.jira_manager.update_cached_jira_project_data,
pause=False),
MenuOption.print_blank_line(),
MenuOption.return_to_previous_menu(self.go_to_main_menu)
]
self.active_menu = self.main_menu # type: List[MenuOption]
self.menu_header = 'uninit' # type: str
self.go_to_main_menu()
self._load_config()
# let user read startup info
pause()