def collect_existing_github_projects(organization: KippoOrganization,
as_user: KippoUser):
manager = GithubOrganizationManager(
organization=organization.github_organization_name,
token=organization.githubaccesstoken.token)
# get existing html_urls
existing_html_urls = KippoProject.objects.filter(
organization=organization,
github_project_url__isnull=False).values_list('github_project_url',
flat=True)
added_projects = []
for project in manager.projects():
if project.html_url not in existing_html_urls:
# create related KippoProject
kippo_project = KippoProject(
created_by=as_user,
updated_by=as_user,
organization=organization,
name=project.name,
columnset=organization.default_columnset,
github_project_url=project.html_url,
)
kippo_project.save()
added_projects.append(kippo_project)
logger.info(
f'(collect_existing_github_projects) Created KippoProject: {project.name} {project.html_url}'
)
else:
logger.debug(
f'(collect_existing_github_projects) Already Exists SKIPPING: {project.name} {project.html_url}'
)
return added_projects
def create_github_organizational_project_action(modeladmin, request,
queryset) -> None:
"""
Admin Action command to create a github organizational project from the selected KippoProject(s)
Where an existing Github Organization project does not exist (not assigned)
"""
successful_creation_projects = []
skipping = []
for kippo_project in queryset:
if kippo_project.github_project_url:
message = f'{kippo_project.name} already has GitHub Project set ({kippo_project.github_project_url}), SKIPPING!'
logger.warning(message)
skipping.append(message)
else:
if not kippo_project.columnset:
modeladmin.message_user(
request,
message=
f'ProjectColumnSet not defined for {kippo_project}, cannot create Github Project!',
level=messages.ERROR,
)
return
columns = kippo_project.get_column_names()
github_manager = GithubOrganizationManager(
organization=kippo_project.github_organization_name,
token=kippo_project.githubaccesstoken.token)
# create the organizational project in github
# create_organizational_project(organization: str, name: str, description: str, columns: list=None) -> Tuple[str, List[object]]:
url, _ = github_manager.create_organizational_project(
name=kippo_project.github_project_name,
description=kippo_project.github_project_description,
columns=columns,
)
kippo_project.github_project_url = url
kippo_project.save()
successful_creation_projects.append((kippo_project.name, url))
if skipping:
for m in skipping:
modeladmin.message_user(
request,
message=m,
level=messages.WARNING,
)
if successful_creation_projects:
modeladmin.message_user(
request,
message=
f'({len(successful_creation_projects)}) GitHub Projects Created: {successful_creation_projects}',
level=messages.INFO,
)
def update_repository_labels(organization, token, repositories, labels_definition_filepath, delete=False):
"""
Update a repository's labels based on a given definition file.
NOTE:
if delete == True, undefined labels will be removed/deleted
:param organization:
:param token:
:param repositories:
:param labels_definition_filepath:
:param delete:
:return: created_labels, deleted_labels
"""
created_labels = []
deleted_labels = []
assert os.path.exists(labels_definition_filepath)
manager = GithubOrganizationManager(organization,
token)
# load file
label_definitions = None
with open(labels_definition_filepath, 'r', encoding='utf8') as labels_json:
label_definitions = json.load(labels_json)
assert label_definitions
defined_label_names = [label['name'] for label in label_definitions]
repositories = tuple(repositories)
for repository in manager.repositories(names=repositories):
existing_label_names = [label['name']for label in repository.labels]
for label_definition in label_definitions:
repository.create_label(label_definition['name'],
label_definition['description'],
label_definition['color'])
created_labels.append(label_definition)
if delete:
undefined_label_names = set(existing_label_names) - set(defined_label_names)
for label_name in undefined_label_names:
repository.delete_label(label_name)
deleted_labels.append(label_name)
return created_labels, deleted_labels
def __init__(self, organization,
projects,
milestone_start_dates=None,
holiday_calendar=None,
personal_holidays=None,
phantom_user_count=0,
start_date=None,
milestone_start_date_now=False):
token = os.environ.get('GITHUB_OAUTH_TOKEN', None)
if not token:
raise MissingRequiredEnvironmentVariable('Required GITHUB_OAUTH_TOKEN EnVar not set!')
self.project_manager = GithubOrganizationManager(token, organization)
self.projects = projects # organizational projects name
self.milestone_start_dates = milestone_start_dates if milestone_start_dates is not None else {}
self.holiday_calendar = holiday_calendar
self.personal_holidays = personal_holidays
self.phantom_user_count = phantom_user_count
self.start_date = start_date
self.fallback_milestone_start_date = None
if milestone_start_date_now:
self.fallback_milestone_start_date = arrow.utcnow().date()
self.milestones = {}
self._tasks = None
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-t', '--token',
default=os.environ.get('GITHUB_OAUTH_TOKEN', None),
help='GITHUB OAUTH Token')
parser.add_argument('-o', '--organization',
default='abeja-inc',
help='Github Organization Name')
parser.add_argument('-r', '--repository',
default=None,
help='Project Name Filter(s) [DEFAULT=None]')
parser.add_argument('--verbose',
action='store_true',
default=False,
help='If given, DEBUG info will be displayed')
subparsers = parser.add_subparsers()
milestone_parser = subparsers.add_parser('miletones')
milestone_parser.add_argument('-d', '--dump',
action='store_true',
default=False,
help='Dump MILESTONES for the given repoistory')
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
manager = GithubOrganizationManager(args.organization,
args.token)
repository = next(manager.repositories(name=args.repository)) # should only be 1
print(repository.milestones)
class GithubOrganizationProjectsAdaptor:
"""Class provides an adaptor for github organization projects to
qlu Task/TaskEstimates/Milestones
NOTE: Only 1 project supported
"""
def __init__(self, organization,
projects,
milestone_start_dates=None,
holiday_calendar=None,
personal_holidays=None,
phantom_user_count=0,
start_date=None,
milestone_start_date_now=False):
token = os.environ.get('GITHUB_OAUTH_TOKEN', None)
if not token:
raise MissingRequiredEnvironmentVariable('Required GITHUB_OAUTH_TOKEN EnVar not set!')
self.project_manager = GithubOrganizationManager(token, organization)
self.projects = projects # organizational projects name
self.milestone_start_dates = milestone_start_dates if milestone_start_dates is not None else {}
self.holiday_calendar = holiday_calendar
self.personal_holidays = personal_holidays
self.phantom_user_count = phantom_user_count
self.start_date = start_date
self.fallback_milestone_start_date = None
if milestone_start_date_now:
self.fallback_milestone_start_date = arrow.utcnow().date()
self.milestones = {}
self._tasks = None
def _collect_tasks(self):
"""Collect Issues from projects and convert to qlu Task objects
:return:
"""
tasks = []
issue_state_index = 3
issue_url_index = 5
projects_processed = []
available_projects = []
for project in self.project_manager.projects():
available_projects.append(project.name)
if project.name in self.projects:
for issue_object in project.issues():
if not issue_object.milestone:
msg = (f'Milestone not assigned to Issue({issue_object.id}) [{issue_object.url}], '
f'all Issues must be assigned to a Milestone!')
raise MissingQluMilestone(msg)
issue = issue_object.simple
issue_url = issue[issue_url_index]
if issue[issue_state_index] != 'open':
warnings.warn('Issue not in open STATE, SKIPPING: {}'.format(issue_url))
continue
# convert issue to qlu task!
# get labels from task to define the estimates
# --> See QLU_GITHUB_ESTIMATE_LABEL_PREFIXES for expected labels
issue_label_index = 7
labels = issue[issue_label_index]
parsed_estimates = {}
for label in labels:
if label.startswith(QLU_GITHUB_ESTIMATE_LABEL_PREFIXES):
estimate_type, value = label.split(QLU_GITHUB_LABEL_SEPARATOR)
parsed_estimates[estimate_type] = int(value)
main_estimate = parsed_estimates.get('estimate', None)
if not main_estimate:
warnings.warn('Expected "estimate:N" label NOT attached: {}'.format(issue_url))
estimates = NO_ESTIMATES
else:
estimate_values = []
for offset, estimate_type in enumerate(QLU_GITHUB_ESTIMATE_LABEL_PREFIXES, -1):
fallback_value = main_estimate
if offset < 0:
fallback_value = main_estimate/1.2 # min
elif offset > 0:
fallback_value = main_estimate * 1.6 # max
value = parsed_estimates.get(estimate_type, fallback_value)
estimate_values.append(value)
estimates = QluTaskEstimates(*estimate_values)
# priority is based on column position and issue(card) position in the column
issue_column_index = 4
column = issue[issue_column_index]
# expect project matches expected format, should have ALL expected columns
assert column in QLU_GITHUB_COLUMNS
# Only process issues in ACTIVE columns
column_priority_index = 15
absolute_priority = None
for p, column_name in enumerate(QLU_GITHUB_ACTIVE_COLUMN_PRIORITY, 1):
base_priority = p * 1000
if column in QLU_GITHUB_ACTIVE_COLUMN_PRIORITY:
column_priority = issue[column_priority_index]
absolute_priority = base_priority + column_priority
if absolute_priority is None:
warnings.warn('Issue not in Project ACTIVE COLUMN({}), SKIPPING: {}'.format(QLU_GITHUB_ACTIVE_COLUMN_PRIORITY,
issue_object.html_url))
continue
task = QluTask(
issue_object.id,
absolute_priority,
issue_object.depends_on,
estimates,
[a.login for a in issue_object.assignees],
issue_object._project.name,
issue_object.milestone.title,
)
# Note Github milestone's don't have a start_date component!
milestone_start_date = self.milestone_start_dates.get(issue_object.milestone.title,
self.fallback_milestone_start_date)
github_milestone = QluMilestone(issue_object.milestone.title,
milestone_start_date,
arrow.get(issue_object.milestone.due_on).date())
self.milestones[issue_object.milestone.title] = github_milestone
tasks.append(task)
projects_processed.append(project.name)
for p in self.projects:
if p not in projects_processed:
raise InvalidGithubOrganizationProject('Project "{}" not in: {}'.format(p, available_projects))
self._tasks = tasks
print('_tasks:', self._tasks)
return tasks
def _collect_milestones(self):
"""
Collect Milestones from projects and convert to qlu Milestone objects
:return:
"""
# TODO: properly support milestone handling
return self.milestones.values()
def generate_task_scheduler(self):
tasks = self._collect_tasks()
milestones = list(self._collect_milestones())
scheduler = QluTaskScheduler(milestones=milestones,
holiday_calendar=self.holiday_calendar,
assignee_personal_holidays=self.personal_holidays,
phantom_user_count=self.phantom_user_count,
start_date=self.start_date)
return scheduler.schedule(tasks)
def collect_github_project_issues(
kippo_organization: KippoOrganization,
status_effort_date: datetime.date = None) -> tuple:
"""
1. Collect issues from attached github projects
2. If related KippoTask does not exist, create one
3. If KippoTask exists create KippoTaskStatus
:param kippo_organization: KippoOrganization
:param status_effort_date: Date to get tasks from
:return: processed_projects_count, created_task_count, created_taskstatus_count
"""
# TODO: support non-update of done tasks
# get done tasks for active projects and last week task status
# if *still* in 'done' state do not create a new KippoTaskStatus entry
if not status_effort_date:
status_effort_date = timezone.now().date()
if not kippo_organization.githubaccesstoken or not kippo_organization.githubaccesstoken.token:
raise OrganizationConfigurationError(
f'Token Not configured for: {kippo_organization.name}')
manager = GithubOrganizationManager(
organization=kippo_organization.github_organization_name,
token=kippo_organization.githubaccesstoken.token)
existing_tasks_by_html_url = {
t.github_issue_html_url: t
for t in KippoTask.objects.filter(is_closed=False)
if t.github_issue_html_url
}
existing_open_projects = list(
ActiveKippoProject.objects.filter(github_project_url__isnull=False))
github_users = {
u.github_login: u
for u in KippoUser.objects.filter(github_login__isnull=False)
}
if settings.UNASSIGNED_USER_GITHUB_LOGIN not in github_users:
raise KippoConfigurationError(
f'"{settings.UNASSIGNED_USER_GITHUB_LOGIN}" must be created as a User to manage unassigned tasks'
)
# collect project issues
processed_projects = 0
new_task_count = 0
new_taskstatus_objects = []
for github_project in manager.projects():
logger.info('Processing github project ({})...'.format(
github_project.name))
# get the related KippoProject
# --- For some reason standard filtering was not working as expected, so this method is being used...
# --- The following was only returning a single project, 'Project(SB Mujin)'.
# --- Project.objects.filter(is_closed=False, github_project_url__isnull=False)
kippo_project = None
for candiate_kippo_project in existing_open_projects:
if candiate_kippo_project.github_project_url == github_project.html_url:
kippo_project = candiate_kippo_project
break
if not kippo_project:
logger.info('X -- Kippo Project Not found!')
else:
logger.info('-- KippoProject: {}'.format(kippo_project.name))
processed_projects += 1
logger.info('-- Processing Related Github Issues...')
count = 0
for count, issue in enumerate(github_project.issues(), 1):
# check if issue is open
# refer to github API for available fields
# https://developer.github.com/v3/issues/
if issue.state == 'open':
# add related repository as GithubRepository
repo_api_url = issue.repository_url
repo_html_url = issue.html_url.split('issues')[0]
name_index = -2
issue_repo_name = repo_html_url.rsplit('/', 2)[name_index]
kippo_github_repository, created = GithubRepository.objects.get_or_create(
created_by=GITHUB_MANAGER_USER,
updated_by=GITHUB_MANAGER_USER,
project=kippo_project,
name=issue_repo_name,
api_url=repo_api_url,
html_url=repo_html_url)
if created:
logger.info(
f'>>> Created GithubRepository({kippo_project} {issue_repo_name})!'
)
default_task_category = kippo_github_repository.project.organization.default_task_category
# check if issue exists
existing_task = existing_tasks_by_html_url.get(
issue.html_url, None)
assignees = [
issue_assignee.login
for issue_assignee in issue.assignees
if issue_assignee.login in github_users
]
if not assignees:
# assign task to special 'unassigned' user if task is not assigned to anyone
assignees = [settings.UNASSIGNED_USER_GITHUB_LOGIN]
estimate_denominator = len(assignees)
for issue_assignee in assignees:
issue_assigned_user = github_users.get(
issue_assignee, None)
if not issue_assigned_user:
logger.warning(
f'Not assigned ({issue_assignee}): {issue.html_url}'
)
else:
# only add task if issue is assigned to someone in the system!
if not existing_task:
category = get_github_issue_category_label(
issue)
if not category:
category = default_task_category
existing_task = KippoTask(
created_by=GITHUB_MANAGER_USER,
updated_by=GITHUB_MANAGER_USER,
title=issue.title,
category=category,
project=kippo_project,
assignee=issue_assigned_user,
github_issue_api_url=issue.url,
github_issue_html_url=issue.html_url,
description=issue.body,
)
existing_task.save()
new_task_count += 1
logger.info(
f'--> Created KippoTask: {issue.title} ({issue_assigned_user.username})'
)
# only update status if active or done (want to pick up
# -- this condition is only met when the task is open, closed tasks will not be updated.
active_task_column_names = kippo_project.columnset.get_active_column_names(
)
done_task_column_names = kippo_project.columnset.get_done_column_names(
)
task_status_updates_states = active_task_column_names + done_task_column_names
if issue.project_column in task_status_updates_states:
latest_comment = ''
if issue.latest_comment_body:
latest_comment = f'{issue.latest_comment_created_by} [ {issue.latest_comment_created_at} ] {issue.latest_comment_body}'
unadjusted_issue_estimate = get_github_issue_estimate_label(
issue)
adjusted_issue_estimate = None
if unadjusted_issue_estimate:
# adjusting to take into account the number of assignees working on it
# -- divides task load by the number of assignees
adjusted_issue_estimate = unadjusted_issue_estimate / estimate_denominator
# create KippoTaskStatus with updated estimate
status = KippoTaskStatus(
task=existing_task,
created_by=GITHUB_MANAGER_USER,
updated_by=GITHUB_MANAGER_USER,
state=issue.project_column,
estimate_days=adjusted_issue_estimate,
effort_date=status_effort_date,
comment=latest_comment)
try:
status.save()
new_taskstatus_objects.append(status)
logger.info(
f'--> KippoTaskStatus Added: {issue.title} ({status_effort_date})'
)
except IntegrityError as e:
logger.warning(
f'--> KippoTaskStatus Already Exists: {issue.title} ({status_effort_date})'
)
logger.warning(str(e))
logger.info(
f'>>> {kippo_project.name} - processed issues: {count}')
return processed_projects, new_task_count, len(new_taskstatus_objects)
def update_github_milestones(self,
close=False) -> List[Tuple[bool, object]]:
"""
Create or Update related github milestones belonging to github repositories attached to the related project.
:return:
.. code:: python
[
(CREATED, GithubMilestone Object),
]
"""
github_milestones = []
# collect existing
existing_github_milestones_by_repo_html_url = {}
existing_github_repositories_by_html_url = {}
for github_repository in GithubRepository.objects.filter(
project=self.project):
url = github_repository.html_url
if url.endswith('/'):
# remove to match returned result from github
url = url[:-1]
existing_github_repositories_by_html_url[url] = github_repository
for github_milestone in GithubMilestone.objects.filter(
repository=github_repository):
existing_github_milestones_by_repo_html_url[
url] = github_milestone
github_organization_name = self.project.organization.github_organization_name
token = self.project.organization.githubaccesstoken.token
manager = GithubOrganizationManager(
organization=github_organization_name, token=token)
# identify related github project and get related repository urls
related_repository_html_urls = list(
existing_github_repositories_by_html_url.keys())
if not related_repository_html_urls:
logger.warning(
f'Related Repository URLS not found for Telos Project: {self.project.name}'
)
else:
for repository in manager.repositories():
if repository.html_url in related_repository_html_urls:
print(f'Updating {repository.name} Milestones...')
created = False
github_state = self.github_state
if close:
github_state = GITHUB_MILESTONE_CLOSE_STATE
if repository.html_url in existing_github_milestones_by_repo_html_url:
github_milestone = existing_github_milestones_by_repo_html_url[
repository.html_url]
_ = repository.update_milestone(
title=self.title,
description=self.description,
due_on=self.target_date,
state=github_state,
number=github_milestone.number)
else:
# create
response = repository.create_milestone(
title=self.title,
description=self.description,
due_on=self.target_date,
state=github_state)
# get number and create GithubMilestone entry
# milestone_content defined at:
# https://developer.github.com/v3/issues/milestones/#create-a-milestone
_, milestone_content = response
number = milestone_content['number']
api_url = milestone_content['url']
html_url = milestone_content['html_url']
github_repository = existing_github_repositories_by_html_url[
repository.html_url]
github_milestone = GithubMilestone(
milestone=self,
number=number,
repository=github_repository,
api_url=api_url,
html_url=html_url)
github_milestone.save()
created = True
action = 'create' if created else 'update'
print(
f'+ {action} Github Milestone: ({repository.name}) {self.title}'
)
github_milestones.append((created, github_milestone))
return github_milestones