def get_project_id_from_github_repository(github_repository_id): # Get repository ID that references the github ID. try: repository = Repository().get_repository_by_external_id(github_repository_id, 'github') except DoesNotExist: return None # Get project ID (contract group ID) of this repository return repository.get_repository_project_id()
def delete_organization(auth_user, organization_name): """ Deletes a github organization based on Name. :param organization_name: The Name of the github organization. :type organization_name: Name """ # Retrieve SFDC ID for this organization github_organization = get_github_organization_instance() try: github_organization.load(str(organization_name)) except DoesNotExist as err: cla.log.warning('organization does not exist: {} - unable to delete'.format(organization_name)) return {'errors': {'organization_name': str(err)}} organization_sfid = github_organization.get_organization_sfid() # Validate user is authorized for this SFDC ID. can_access = check_user_authorization(auth_user, organization_sfid) if not can_access['valid']: return can_access['errors'] # Find all repositories that are under this organization repositories = Repository().get_repositories_by_organization(organization_name) for repository in repositories: repository.delete() github_organization.delete() return {'success': True}
def get_installation_id_from_github_repository(github_repository_id): # Get repository ID that references the github ID. try: repository = Repository().get_repository_by_external_id(github_repository_id, 'github') except DoesNotExist: return None # Get Organization from this repository organization = GitHubOrg() try: organization.load(repository.get_repository_organization_name()) except DoesNotExist: return None # Get this organization's installation ID return organization.get_organization_installation_id()
def handle_installation_repositories_removed_event(action: str, body: dict): func_name = 'github.activity.handle_installation_repositories_removed_event' # Who triggered the event user_login = body['sender']['login'] cla.log.debug( f'{func_name} - processing github [installation_repositories] ' f'activity {action} callback created by GitHub user {user_login}.') repository_removed = body['repositories_removed'] repositories = [] for repo in repository_removed: repository_external_id = repo['id'] ghrepo = Repository().get_repository_by_external_id( repository_external_id, 'github') if ghrepo is not None: repositories.append(ghrepo) # Notify the Project Managers that the following list of repositories were removed notify_project_managers(repositories) # The following list of repositories were deleted/removed from GitHub - we need to remove # the repo entry from our repos table for repo in repositories: project_model = get_project_instance() try: project_model.load(project_id=repo.get_repository_project_id()) except DoesNotExist as err: cla.log.warning( f'{func_name} - unable to load project (cla_group) by ' f'project_id: {repo.get_repository_project_id()}, error: {err}' ) msg = ( f'Disabling repository {repo.get_repository_name()} ' f'from GitHub organization : {repo.get_repository_organization_name()} ' f'with URL: {repo.get_repository_url()} ' 'from the CLA configuration.') cla.log.debug(msg) # Disable the repo and add a note repo.set_enabled(False) repo.add_note(f'{datetime.now()} - Disabling repository due to ' 'GitHub installation_repositories delete event ' f'for CLA Group {project_model.get_project_name()}') repo.save() # Log the event Event.create_event( event_type=EventType.RepositoryDisable, event_project_id=repo.get_repository_project_id(), event_project_name=project_model.get_project_name(), event_company_id=None, event_data=msg, event_summary=msg, event_user_id=user_login, contains_pii=False, )
def handle_installation_event(action: str, body: dict): func_name = 'github.activity.handle_installation_event' cla.log.debug(f'{func_name} - processing github [installation] activity callback...') # New Installations if action == 'created': cla.log.debug(f'{func_name} - processing github installation activity for action: {action}') org_name = get_org_name_from_installation_event(body) if org_name is None: cla.log.warning(f'{func_name} - Unable to determine organization name from the github installation event ' f'with action: {action}' f'event body: {json.dumps(body)}') return {'status', f'GitHub installation {action} event malformed.'} cla.log.debug(f'Locating organization using name: {org_name}') existing = get_organization(org_name) if 'errors' in existing: cla.log.warning(f'{func_name} - Received github installation created event for organization: {org_name}, ' 'but the organization is not configured in EasyCLA') # TODO: Need a way of keeping track of new organizations that don't have projects yet. return {'status': 'Github Organization must be created through the Project Management Console.'} elif not existing['organization_installation_id']: update_organization( existing['organization_name'], existing['organization_sfid'], body['installation']['id'], ) cla.log.info(f'{func_name} - Organization enrollment completed: {existing["organization_name"]}') return {'status': 'Organization Enrollment Completed. CLA System is operational'} else: cla.log.info(f'{func_name} - Organization already enrolled: {existing["organization_name"]}') cla.log.info(f'{func_name} - Updating installation ID for ' f'github organization: {existing["organization_name"]}') update_organization( existing['organization_name'], existing['organization_sfid'], body['installation']['id'], ) return {'status': 'Already Enrolled Organization Updated. CLA System is operational'} elif action == 'deleted': cla.log.debug(f'{func_name} - processing github installation activity for action: {action}') org_name = get_org_name_from_installation_event(body) if org_name is None: cla.log.warning('Unable to determine organization name from the github installation event ' f'with action: {action}' f'event body: {json.dumps(body)}') return {'status', f'GitHub installation {action} event malformed.'} repositories = Repository().get_repositories_by_organization(org_name) notify_project_managers(repositories) return else: cla.log.debug(f'{func_name} - ignoring github installation activity for action: {action}')
def get_sfdc_project_repositories(project): """ Gets all SFDC repositories and divide them for current contract group and other contract groups :param project: The Project object :type project: Project :return: array of all sfdc project repositories :rtype: dict """ # Get all SFDC Project repositories sfdc_id = project.get_project_external_id() all_project_repositories = Repository().get_repository_by_sfdc_id(sfdc_id) return [repo.to_dict() for repo in all_project_repositories]
def redirect_to_console(self, installation_id, repository_id, pull_request_id, redirect, request): console_endpoint = cla.conf['CONTRIBUTOR_BASE_URL'] # Get repository using github's repository ID. repository = Repository().get_repository_by_external_id( repository_id, "github") if repository is None: cla.log.warning( 'Could not find repository with the following repository_id: %s', repository_id) return None # Get project ID from this repository project_id = repository.get_repository_project_id() user = self.get_or_create_user(request) # Ensure user actually requires a signature for this project. # TODO: Skipping this for now - we can do this for ICLAs but there's no easy way of doing # the check for CCLAs as we need to know in advance what the company_id is that we're checking # the CCLA signature for. # We'll have to create a function that fetches the latest CCLA regardless of company_id. # icla_signature = cla.utils.get_user_signature_by_github_repository(installation_id, user) # ccla_signature = cla.utils.get_user_signature_by_github_repository(installation_id, user, company_id=?) # try: # document = cla.utils.get_project_latest_individual_document(project_id) # except DoesNotExist: # cla.log.debug('No ICLA for project %s' %project_id) # if signature is not None and \ # signature.get_signature_document_major_version() == document.get_document_major_version(): # return cla.utils.redirect_user_by_signature(user, signature) # Store repository and PR info so we can redirect the user back later. cla.utils.set_active_signature_metadata(user.get_user_id(), project_id, repository_id, pull_request_id) # Generate console URL console_url = 'https://' + console_endpoint + \ '/#/cla/project/' + project_id + \ '/user/' + user.get_user_id() + \ '?redirect=' + redirect raise falcon.HTTPFound(console_url)
def delete_repository(repository_id): """ Deletes a repository based on ID. :param repository_id: The ID of the repository. :type repository_id: ID """ repository = Repository() try: repository.load(str(repository_id)) except DoesNotExist as err: return {'errors': {'repository_id': str(err)}} repository.delete() return {'success': True}
def handle_installation_repositories_added_event(action: str, body: dict): func_name = 'github.activity.handle_installation_repositories_added_event' # Who triggered the event user_login = body['sender']['login'] cla.log.debug(f'{func_name} - processing github [installation_repositories] ' f'activity {action} callback created by GitHub user {user_login}.') # Grab the list of repositories added from the event model repository_added = body.get('repositories_added', []) # Create a unique list of repositories for the email that we need to send out repository_list = set([repo.get('full_name', None) for repo in repository_added]) # All the repos in the message should be under the same GitHub Organization organization_name = '' for repo in repository_added: # Grab the information repository_external_id = repo['id'] # example: 271841254 repository_name = repo['name'] # example: PyImath repository_full_name = repo['full_name'] # example: AcademySoftwareFoundation/PyImath organization_name = repository_full_name.split('/')[0] # example: AcademySoftwareFoundation # repository_private = repo['private'] # example: False # Lookup the GitHub Organization in our table - should be there already cla.log.debug(f'{func_name} - Locating organization using name: {organization_name}') org_model = get_organization_model(organization_name) if org_model is None: # Should we create since it's missing? cla.log.warning(f'Unable to locate GitHub Organization {organization_name} in our database') continue # Should we update to ensure the installation_id is set? if org_model.get_organization_installation_id() is None: # Update the installation ID org_model.set_organization_installation_id(body.get('installation', {}).get('id', None)) org_model.save() # Check to see if the auto enabled flag is set if org_model.get_auto_enabled(): # We need to check that we only have 1 CLA Group - auto-enable only works when the entire # Organization falls under a single CLA Group - otherwise, how would we know which CLA Group # to add them to? First we query all the existing repositories associated with this Github Org - # they should all point the the single CLA Group - let's verify this... existing_github_repositories = Repository().get_repositories_by_organization(organization_name) cla_group_ids = set(()) # hoping for only 1 unique value - set collection discards duplicates cla_group_repo_sfids = set(()) # keep track of the existing SFDC IDs from existing repos for existing_repo in existing_github_repositories: cla_group_ids.add(existing_repo.get_repository_project_id()) cla_group_repo_sfids.add(existing_repo.get_repository_sfdc_id()) # We should only have one... if len(cla_group_ids) != 1 or len(cla_group_repo_sfids) != 1: cla.log.warning(f'{func_name} - Auto Enabled set for Organization {organization_name}, ' f'but we found repositories or SFIDs that belong to multiple CLA Groups. ' 'Auto Enable only works when all repositories under a given ' 'GitHub Organization are associated with a single CLA Group. This ' f'organization is associated with {len(cla_group_ids)} CLA Groups and ' f'{len(cla_group_repo_sfids)} SFIDs.') return cla_group_id = cla_group_ids.pop() project_model = get_project_instance() try: project_model.load(project_id=cla_group_id) except DoesNotExist as err: cla.log.warning(f'{func_name} - unable to load project (cla_group) by ' f'project_id: {cla_group_id}, error: {err}') cla.log.debug(f'{func_name} - Organization {organization_name} has auto_enabled set - ' f'adding repository: {repository_name} to ' f'CLA Group: {project_model.get_project_name()}') try: # Create the new repository entry and associate it with the CLA Group new_repository = Repository( repository_id=str(uuid.uuid4()), repository_project_id=cla_group_id, repository_name=repository_full_name, repository_type='github', repository_url='https://github.com/' + repository_full_name, repository_organization_name=organization_name, repository_external_id=repository_external_id, repository_sfdc_id=cla_group_repo_sfids.pop(), ) new_repository.set_enabled(True) new_repository.save() # Log the event msg = (f'Adding repository {repository_full_name} ' f'from GitHub organization : {organization_name} ' f'with URL: https://github.com/{repository_full_name} ' 'to the CLA configuration. GitHub organization was set to auto-enable.') Event.create_event( event_type=EventType.RepositoryAdded, event_project_id=cla_group_id, event_project_name=project_model.get_project_name(), event_company_id=None, event_data=msg, event_summary=msg, event_user_id=user_login, contains_pii=False, ) except Exception as err: cla.log.warning(f'{func_name} - Could not create GitHub repository: {err}') return else: cla.log.debug(f'{func_name} - Auto enabled NOT set for GitHub Organization {organization_name} - ' f'not auto-adding repository: {repository_full_name}') return # Notify the Project Managers notify_project_managers_auto_enabled(organization_name, repository_list)
def update_change_request(self, installation_id, github_repository_id, change_request_id): # Queries GH for the complete pull request details, see: # https://developer.github.com/v3/pulls/#response-1 pull_request = self.get_pull_request(github_repository_id, change_request_id, installation_id) cla.log.debug(f'Retrieved pull request: {pull_request}') # Get all unique users/authors involved in this PR - returns a list of # (commit_sha_string, (author_id, author_username, author_email) tuples commit_authors = get_pull_request_commit_authors(pull_request) try: # Get existing repository info using the repository's external ID, # which is the repository ID assigned by github. cla.log.debug( f'PR: {pull_request.number}, Loading GitHub repository by id: {github_repository_id}' ) repository = Repository().get_repository_by_external_id( github_repository_id, "github") if repository is None: cla.log.warning( f'PR: {pull_request.number}, Failed to load GitHub repository by ' f'id: {github_repository_id} in our DB - repository reference is None - ' 'Is this org/repo configured in the Project Console?' ' Unable to update status.') return except DoesNotExist: cla.log.warning( f'PR: {pull_request.number}, could not find repository with the ' f'repository ID: {github_repository_id}') cla.log.warning( f'PR: {pull_request.number}, failed to update change request of ' f'repository {github_repository_id} - returning') return # Get Github Organization name that the repository is configured to. organization_name = repository.get_repository_organization_name() cla.log.debug('PR: {}, determined github organization is: {}'.format( pull_request.number, organization_name)) # Check that the Github Organization exists. github_org = GitHubOrg() try: github_org.load(organization_name) except DoesNotExist: cla.log.warning( 'PR: {}, Could not find Github Organization with the following organization name: {}' .format(pull_request.number, organization_name)) cla.log.warning( 'PR: {}, Failed to update change request of repository {} - returning' .format(pull_request.number, github_repository_id)) return # Ensure that installation ID for this organization matches the given installation ID if github_org.get_organization_installation_id() != installation_id: cla.log.warning( 'PR: {}, the installation ID: {} of this organization does not match ' 'installation ID: {} given by the pull request.'.format( pull_request.number, github_org.get_organization_installation_id(), installation_id)) cla.log.error( 'PR: {}, Failed to update change request of repository {} - returning' .format(pull_request.number, github_repository_id)) return # Retrieve project ID from the repository. project_id = repository.get_repository_project_id() # Find users who have signed and who have not signed. signed = [] missing = [] cla.log.debug( f'PR: {pull_request.number}, scanning users - determining who has signed a CLA an who has not.' ) for commit_sha, author_info in commit_authors: # Extract the author info tuple details author_id = author_info[0] author_username = author_info[1] author_email = author_info[2] cla.log.debug( 'PR: {}, processing sha: {} from author id: {}, username: {}, email: {}' .format(pull_request.number, commit_sha, author_id, author_username, author_email)) handle_commit_from_user(project_id, commit_sha, author_info, signed, missing) cla.log.debug( 'PR: {}, updating github pull request for repo: {}, ' 'with signed authors: {} with missing authors: {}'.format( pull_request.number, github_repository_id, signed, missing)) update_pull_request(installation_id, github_repository_id, pull_request, signed=signed, missing=missing)
def create_repository(auth_user: AuthUser, # pylint: disable=too-many-arguments repository_project_id, repository_name, repository_organization_name, repository_type, repository_url, repository_external_id=None): """ Creates a repository and returns the newly created repository in dict format. :param repository_project_id: The ID of the repository project. :type repository_project_id: string :param repository_name: The new repository name. :type repository_name: string :param repository_type: The new repository type ('github', 'gerrit', etc). :type repository_organization_name: string :param repository_organization_name: The repository organization name :type repository_type: string :param repository_url: The new repository URL. :type repository_url: string :param repository_external_id: The ID of the repository from the repository provider. :type repository_external_id: string :return: dict representation of the new repository object. :rtype: dict """ # Check that organization exists github_organization = GitHubOrg() try: github_organization.load(str(repository_organization_name)) except DoesNotExist as err: return {'errors': {'organization_name': str(err)}} # Check that project is valid. project = Project() try: project.load(str(repository_project_id)) except DoesNotExist as err: return {'errors': {'repository_project_id': str(err)}} # Get SFDC project identifier sfdc_id = project.get_project_external_id() # Validate user is authorized for this project can_access = cla.controllers.project.check_user_authorization(auth_user, sfdc_id) if not can_access['valid']: return can_access['errors'] # Validate if exist already repository linked to a contract group if repository_external_id is not None: # Seach for the repository linked_repository = Repository().get_repository_by_external_id(repository_external_id, repository_type) # If found return an error if linked_repository is not None: return {'errors': {'repository_external_id': 'This repository is alredy configured for a contract group.'}} repository = Repository() repository.set_repository_id(str(uuid.uuid4())) repository.set_repository_project_id(str(repository_project_id)) repository.set_repository_sfdc_id(str(sfdc_id)) repository.set_repository_name(repository_name) repository.set_repository_organization_name(repository_organization_name) repository.set_repository_type(repository_type) repository.set_repository_url(repository_url) if repository_external_id is not None: repository.set_repository_external_id(repository_external_id) repository.save() return repository.to_dict()
def update_repository(repository_id, # pylint: disable=too-many-arguments repository_project_id=None, repository_type=None, repository_name=None, repository_url=None, repository_external_id=None): """ Updates a repository and returns the newly updated repository in dict format. Values of None means the field will not be updated. :param repository_id: ID of the repository to update. :type repository_id: ID :param repository_project_id: ID of the repository project. :type repository_project_id: string :param repository_name: New name for the repository. :type repository_name: string | None :param repository_type: New type for repository ('github', 'gerrit', etc). :type repository_type: string | None :param repository_url: New URL for the repository. :type repository_url: string | None :param repository_external_id: ID of the repository from the service provider. :type repository_external_id: string :return: dict representation of the repository object. :rtype: dict """ repository = Repository() try: repository.load(str(repository_id)) except DoesNotExist as err: return {'errors': {'repository_id': str(err)}} # TODO: Ensure project_id exists. if repository_project_id is not None: repository.set_repository_project_id(str(repository_project_id)) if repository_type is not None: supported_repo_types = get_supported_repository_providers().keys() if repository_type in supported_repo_types: repository.set_repository_type(repository_type) else: return {'errors': {'repository_type': 'Invalid value passed. The accepted values are: (%s)' \ %'|'.join(supported_repo_types)}} if repository_external_id is not None: # Find a repository is already linked with this external_id linked_repository = Repository().get_repository_by_external_id(repository_external_id, repository.get_repository_type()) # If found return an error if linked_repository is not None: return {'errors': {'repository_external_id': 'This repository is alredy configured for a contract group.'}} repository.set_repository_external_id(repository_external_id) if repository_name is not None: repository.set_repository_name(repository_name) if repository_url is not None: try: val = cla.hug_types.url(repository_url) repository.set_repository_url(val) except ValueError as err: return {'errors': {'repository_url': 'Invalid URL specified'}} repository.save() return repository.to_dict()
def update_change_request(self, installation_id, github_repository_id, change_request_id): pull_request = self.get_pull_request(github_repository_id, change_request_id, installation_id) # Get all unique users involved in this PR. commit_authors = get_pull_request_commit_authors(pull_request) # Get existing repository info using the repository's external ID, which is the repository ID assigned by github. try: repository = Repository().get_repository_by_external_id( github_repository_id, "github") except DoesNotExist: cla.log.error( 'Could not find repository with the repository ID: %s', github_repository_id) cla.log.error( 'Failed to update change request %s of repository %s', change_request_id, github_repository_id) return # Get Github Organization name that the repository is configured to. organization_name = repository.get_repository_organization_name() # Check that the Github Organization exists. github_org = GitHubOrg() try: github_org.load(organization_name) except DoesNotExist: cla.log.error( 'Could not find Github Organization with the following organization name: %s', organization_name) cla.log.error( 'Failed to update change request %s of repository %s', change_request_id, github_repository_id) return # Ensure that installation ID for this organization matches the given installation ID if github_org.get_organization_installation_id() != installation_id: cla.log.error( 'The installation ID: %s of this organization does not match installation ID: %s given by the pull request.', github_org.get_organization_installation_id(), installation_id) cla.log.error( 'Failed to update change request %s of repository %s', change_request_id, github_repository_id) return # Retrieve project ID from the repository. project_id = repository.get_repository_project_id() # Find users who have signed and who have not signed. signed = [] missing = [] for commit, commit_author in commit_authors: # cla.log.info("Author: " + commit_author) if isinstance(commit_author, github.NamedUser.NamedUser): # Handle GitHub user. cla.log.info("Handle GitHub user") handle_commit_from_github_user(project_id, commit, commit_author, signed, missing) elif isinstance(commit_author, github.GitAuthor.GitAuthor): # Handle non-github user (just email and name in commit). cla.log.info( "Handle non-github user (just email and name in commit)") handle_commit_from_git_author(project_id, commit, commit_author, signed, missing) else: # Couldn't find any author information. cla.log.info("Couldn't find any author information.") if commit_author is not None: missing.append((commit.sha, commit_author)) else: missing.append((commit.sha, None)) update_pull_request(installation_id, github_repository_id, pull_request, signed=signed, missing=missing)
def activity(event_type, body): """ Processes the GitHub activity event. :param body: the webhook body payload :type body: dict """ cla.log.debug('github.activity - received github activity event of type %s',event_type) cla.log.debug('github.activity - received github activity event, ' f'action: {get_github_activity_action(body)}...') # If we have the GitHub debug flag set/on... if bool(os.environ.get('GH_APP_DEBUG', '')): cla.log.debug(f'github.activity - body: {json.dumps(body)}') # GitHub Application Installation Event if event_type == 'installation': cla.log.debug('github.activity - processing github installation activity callback...') # New Installations if 'action' in body and body['action'] == 'created': org_name = get_org_name_from_installation_event(body) if org_name is None: cla.log.warning('Unable to determine organization name from the github installation event ' f'with action: {body["action"]}' f'event body: {json.dumps(body)}') return {'status', f'GitHub installation {body["action"]} event malformed.'} cla.log.debug(f'Locating organization using name: {org_name}') existing = get_organization(org_name) if 'errors' in existing: cla.log.warning(f'Received github installation created event for organization: {org_name}, but ' 'the organization is not configured in EasyCLA') # TODO: Need a way of keeping track of new organizations that don't have projects yet. return {'status': 'Github Organization must be created through the Project Management Console.'} elif not existing['organization_installation_id']: update_organization( existing['organization_name'], existing['organization_sfid'], body['installation']['id'], ) cla.log.info('github.activity - Organization enrollment completed: %s', existing['organization_name']) return {'status': 'Organization Enrollment Completed. CLA System is operational'} else: cla.log.info('github.activity - Organization already enrolled: %s', existing['organization_name']) return {'status': 'Organization already enrolled in the CLA system'} elif 'action' in body and body['action'] == 'deleted': cla.log.debug('github.activity - processing github installation activity [deleted] callback...') org_name = get_org_name_from_installation_event(body) if org_name is None: cla.log.warning('Unable to determine organization name from the github installation event ' f'with action: {body["action"]}' f'event body: {json.dumps(body)}') return {'status', f'GitHub installation {body["action"]} event malformed.'} repositories = Repository().get_repositories_by_organization(org_name) notify_project_managers(repositories) return else: pass # GitHub Pull Request Event if event_type == 'pull_request': cla.log.debug('github.activity - processing github pull_request activity callback...') # New PR opened if body['action'] == 'opened' or body['action'] == 'reopened' or body['action'] == 'synchronize': # Copied from repository_service.py service = cla.utils.get_repository_service('github') result = service.received_activity(body) return result if event_type == 'installation_repositories': cla.log.debug('github.activity - processing github installation_repositories activity callback...') if body['action'] == 'removed': repository_removed = body['repositories_removed'] repositories = [] for repo in repository_removed: repository_external_id = repo['id'] ghrepo = Repository().get_repository_by_external_id(repository_external_id,'github') if ghrepo is not None: repositories.append(ghrepo) notify_project_managers(repositories) return cla.log.debug('github.activity - ignoring github activity event, ' f'action: {get_github_activity_action(body)}...')