def from_stashy(klass, repository, labor_hours=True): """ Handles crafting Code.gov Project for Bitbucket Server repositories """ # if not isinstance(repository, stashy.repos.Repository): # raise TypeError('Repository must be a stashy Repository object') if not isinstance(repository, dict): raise TypeError("Repository must be a dict") project = klass() logger.debug( "Stashy: project_key=%s repository_slug=%s", repository["name"], repository["project"]["key"], ) # -- REQUIRED FIELDS -- project["name"] = repository["name"] clone_urls = [clone["href"] for clone in repository["links"]["clone"]] for url in clone_urls: # Only rely on SSH Urls for repository urls if url.startswith("ssh://"): project["repositoryURL"] = url break description = repository["project"].get("description", "") if description: project["description"] = "Project description: %s" % description project["permissions"]["licenses"] = None web_url = repository["links"]["self"][0]["href"] public_server = web_url.startswith("https://bitbucket.org") if repository["public"] and public_server: project["permissions"]["usageType"] = "openSource" if labor_hours: project["laborHours"] = labor_hours_from_url( project["repositoryURL"]) else: project["laborHours"] = 0 project["tags"] = ["bitbucket"] project["contact"]["email"] = "" project["contact"]["URL"] = repository["links"]["self"][0]["href"] # -- OPTIONAL FIELDS -- # project['version'] = '' # project['organization'] = organization.name # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project["status"] = "Development" project["vcs"] = repository["scmId"] project["homepageURL"] = repository["links"]["self"][0]["href"] # project['downloadURL'] = # project['languages'] = # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] # date: [object] A date object describing the release. Empty if repo has no commits. # created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format. # lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format. if repository.get("created", None): project["date"] = { "created": repository["created"], "lastModified": repository["lastModified"], } _prune_dict_null_str(project) return project
def from_gitlab(klass, repository, labor_hours=True, fetch_languages=False): """ Create CodeGovProject object from GitLab Repository """ if not isinstance(repository, gitlab.v4.objects.Project): raise TypeError("Repository must be a gitlab Repository object") project = klass() logger.debug( "GitLab: repository_id=%d path_with_namespace=%s", repository.id, repository.path_with_namespace, ) # -- REQUIRED FIELDS -- project["name"] = repository.name project["repositoryURL"] = repository.http_url_to_repo project["description"] = repository.description # TODO: Update licenses from GitLab API project["permissions"]["licenses"] = None web_url = repository.web_url public_server = web_url.startswith("https://gitlab.com") if repository.visibility in ("public") and public_server: project["permissions"]["usageType"] = "openSource" elif date_parse(repository.created_at) < POLICY_START_DATE: project["permissions"]["usageType"] = "exemptByPolicyDate" if labor_hours: project["laborHours"] = labor_hours_from_url( project["repositoryURL"]) else: project["laborHours"] = 0 project["tags"] = ["gitlab"] + repository.tag_list project["contact"] = {"email": "", "URL": web_url} # -- OPTIONAL FIELDS -- # project['version'] = '' project["organization"] = repository.namespace["name"] # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project["status"] = "Development" project["vcs"] = "git" project["homepageURL"] = repository.web_url api_url = repository.manager.gitlab._url archive_suffix = "/projects/%s/repository/archive" % repository.get_id( ) project["downloadURL"] = api_url + archive_suffix # project['languages'] = [l for l, _ in repository.languages()] if fetch_languages: project["languages"] = [*repository.languages()] # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] project["date"] = { "created": date_parse(repository.created_at).date().isoformat(), "lastModified": date_parse(repository.last_activity_at).date().isoformat(), "metadataLastUpdated": "", } _prune_dict_null_str(project) return project
def from_github3(klass, repository, labor_hours=True): """ Create CodeGovProject object from github3 Repository object """ if not isinstance(repository, github3.repos.repo._Repository): raise TypeError("Repository must be a github3 Repository object") logger.info("Processing: %s", repository.full_name) project = klass() logger.debug("GitHub3: repository=%s", repository) # -- REQUIRED FIELDS -- project["name"] = repository.name project["repositoryURL"] = repository.git_url project["description"] = repository.description try: repo_license = repository.license() except github3.exceptions.NotFoundError: logger.debug("no license found for repo=%s", repository) repo_license = None if repo_license: license = repo_license.license if license: logger.debug("license spdx=%s; url=%s", license.spdx_id, license.url) if license.url is None: project["permissions"]["licenses"] = [{ "name": license.spdx_id }] else: project["permissions"]["licenses"] = [{ "URL": license.url, "name": license.spdx_id }] else: project["permissions"]["licenses"] = None public_server = repository.html_url.startswith("https://github.com") if not repository.private and public_server: project["permissions"]["usageType"] = "openSource" elif date_parse(repository.created_at) < POLICY_START_DATE: project["permissions"]["usageType"] = "exemptByPolicyDate" if labor_hours: project["laborHours"] = labor_hours_from_url( project["repositoryURL"]) else: project["laborHours"] = 0 project["tags"] = ["github"] old_accept = repository.session.headers["Accept"] repository.session.headers[ "Accept"] = "application/vnd.github.mercy-preview+json" topics = repository._get(repository.url + "/topics").json() project["tags"].extend(topics.get("names", [])) repository.session.headers["Accept"] = old_accept # Hacky way to get an Organization object back with GitHub3.py >= 1.2.0 owner_url = repository.owner.url owner_api_response = repository._get(owner_url) organization = repository._json(owner_api_response, 200) project["contact"]["email"] = organization["email"] project["contact"]["URL"] = organization["html_url"] # -- OPTIONAL FIELDS -- # project['version'] = '' project["organization"] = organization["name"] # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project["status"] = "Development" project["vcs"] = "git" project["homepageURL"] = repository.html_url project["downloadURL"] = repository.downloads_url project["languages"] = [l for l, _ in repository.languages()] # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] # date: [object] A date object describing the release. # created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format. # lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format. # metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format. try: created_at = repository.created_at.date() except AttributeError: created_at = date_parse(repository.created_at).date() try: updated_at = repository.updated_at.date() except AttributeError: updated_at = date_parse(repository.updated_at).date() project["date"] = { "created": created_at.isoformat(), "lastModified": updated_at.isoformat(), "metadataLastUpdated": "", } _prune_dict_null_str(project) return project
def from_stashy(klass, repository, labor_hours=True): """ Handles crafting Code.gov Project for Bitbucket Server repositories """ # if not isinstance(repository, stashy.repos.Repository): # raise TypeError('Repository must be a stashy Repository object') if not isinstance(repository, dict): raise TypeError('Repository must be a dict') project = klass() logger.debug( 'Stashy: project_key=%s repository_slug=%s', repository['name'], repository['project']['key'], ) # -- REQUIRED FIELDS -- project['name'] = repository['name'] clone_urls = [clone['href'] for clone in repository['links']['clone']] for url in clone_urls: # Only rely on SSH Urls for repository urls if url.startswith('ssh://'): project['repositoryURL'] = url break description = repository['project'].get('description', '') if description: project['description'] = 'Project description: %s' % description project['permissions']['licenses'] = None web_url = repository['links']['self'][0]['href'] public_server = web_url.startswith('https://bitbucket.org') if repository['public'] and public_server: project['permissions']['usageType'] = 'openSource' if labor_hours: project['laborHours'] = labor_hours_from_url( project['repositoryURL']) else: project['laborHours'] = 0 project['tags'] = ['bitbucket'] project['contact']['email'] = '' project['contact']['URL'] = repository['links']['self'][0]['href'] # -- OPTIONAL FIELDS -- # project['version'] = '' # project['organization'] = organization.name # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project['status'] = 'Development' project['vcs'] = repository['scmId'] project['homepageURL'] = repository['links']['self'][0]['href'] # project['downloadURL'] = # project['languages'] = # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] # date: [object] A date object describing the release. # created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format. # lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format. # metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format. # project['date'] = { # 'created': repository.pushed_at.isoformat(), # 'lastModified': repository.updated_at.isoformat(), # 'metadataLastUpdated': '', # } _prune_dict_null_str(project) return project
def from_gitlab(klass, repository, labor_hours=True): """ Create CodeGovProject object from GitLab Repository """ if not isinstance(repository, gitlab.v4.objects.Project): raise TypeError('Repository must be a gitlab Repository object') project = klass() logger.debug( 'GitLab: repository_id=%d path_with_namespace=%s', repository.id, repository.path_with_namespace, ) # -- REQUIRED FIELDS -- project['name'] = repository.name project['repositoryURL'] = repository.http_url_to_repo project['description'] = repository.description # TODO: Update licenses from GitLab API project['permissions']['license'] = None web_url = repository.web_url public_server = web_url.startswith('https://gitlab.com') if repository.visibility in ('public') and public_server: project['permissions']['usageType'] = 'openSource' elif date_parse(repository.created_at) < POLICY_START_DATE: project['permissions']['usageType'] = 'exemptByPolicyDate' if labor_hours: project['laborHours'] = labor_hours_from_url( project['repositoryURL']) else: project['laborHours'] = 0 project['tags'] = ['gitlab'] + repository.tag_list project['contact'] = { 'email': '', 'URL': web_url, } # -- OPTIONAL FIELDS -- # project['version'] = '' project['organization'] = repository.namespace['name'] # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project['status'] = 'Development' project['vcs'] = 'git' project['homepageURL'] = repository.web_url api_url = repository.manager.gitlab._url archive_suffix = '/projects/%s/repository/archive' % repository.get_id( ) project['downloadURL'] = api_url + archive_suffix # project['languages'] = [l for l, _ in repository.languages()] # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] project['date'] = { 'created': date_parse(repository.created_at).date().isoformat(), 'lastModified': date_parse(repository.last_activity_at).date().isoformat(), 'metadataLastUpdated': '', } _prune_dict_null_str(project) return project
def from_github3(klass, repository, labor_hours=True): """ Create CodeGovProject object from github3 Repository object """ if not isinstance(repository, github3.repos.repo.Repository): raise TypeError('Repository must be a github3 Repository object') logger.info('Processing: %s', repository.full_name) project = klass() logger.debug('GitHub3: repository=%s', repository) # -- REQUIRED FIELDS -- project['name'] = repository.name project['repositoryURL'] = repository.git_url project['description'] = repository.description repo_license = repository.license() if repo_license: license = repo_license.license if license: logger.debug('license spdx=%s; url=%s', license['spdx_id'], license['url']) if license['url'] is None: project['permissions']['licenses'] = [{ "name": license['spdx_id'] }] else: project['permissions']['licenses'] = [{ "URL": license['url'], "name": license['spdx_id'] }] else: project['permissions']['licenses'] = None public_server = repository.html_url.startswith('https://github.com') if not repository.private and public_server: project['permissions']['usageType'] = 'openSource' elif date_parse(repository.created_at) < POLICY_START_DATE: project['permissions']['usageType'] = 'exemptByPolicyDate' if labor_hours: project['laborHours'] = labor_hours_from_url( project['repositoryURL']) else: project['laborHours'] = 0 project['tags'] = ['github'] old_accept = repository.session.headers['Accept'] repository.session.headers[ 'Accept'] = 'application/vnd.github.mercy-preview+json' topics = repository._get(repository.url + '/topics').json() project['tags'].extend(topics['names']) repository.session.headers['Accept'] = old_accept organization = repository.owner project['contact']['email'] = organization.email project['contact']['URL'] = organization.html_url # -- OPTIONAL FIELDS -- # project['version'] = '' project['organization'] = organization.name # TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370 project['status'] = 'Development' project['vcs'] = 'git' project['homepageURL'] = repository.html_url project['downloadURL'] = repository.download_url project['languages'] = [l for l, _ in repository.languages()] # project['partners'] = [] # project['relatedCode'] = [] # project['reusedCode'] = [] # date: [object] A date object describing the release. # created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format. # lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format. # metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format. project['date'] = { 'created': repository.pushed_at.date().isoformat(), 'lastModified': repository.updated_at.date().isoformat(), 'metadataLastUpdated': '', } _prune_dict_null_str(project) return project