def from_tfs(klass, tfs_project, labor_hours=True):
"""
Creates CodeGovProject object from TFS/VSTS/AzureDevOps Instance
"""
project = klass()
project_web_url = ""
# -- REQUIRED FIELDS --
project["name"] = tfs_project.projectInfo.name
if "web" in tfs_project.projectInfo._links.additional_properties:
if "href" in tfs_project.projectInfo._links.additional_properties[
"web"]:
# URL Encodes spaces that are in the Project Name for the Project Web URL
project_web_url = requote_uri(
tfs_project.projectInfo._links.additional_properties["web"]
["href"])
project["repositoryURL"] = project_web_url
project["homepageURL"] = project_web_url
project["description"] = tfs_project.projectInfo.description
project["vcs"] = "TFS/AzureDevOps"
project["permissions"]["license"] = None
project["tags"] = []
if labor_hours:
logger.debug(
"Sorry labor hour calculation not currently supported.")
# project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
else:
project["laborHours"] = 0
if tfs_project.projectCreateInfo.last_update_time < POLICY_START_DATE:
project["permissions"]["usageType"] = "exemptByPolicyDate"
else:
project["permissions"]["usageType"] = "exemptByAgencyMission"
project["permissions"][
"exemptionText"] = "This source code resides on a private server and has not been properly evaluated for releaseability."
project["contact"] = {"email": "", "URL": project_web_url}
project["date"] = {
"lastModified":
tfs_project.projectLastUpdateInfo.last_update_time.date().
isoformat(),
"created":
tfs_project.projectCreateInfo.last_update_time.date().isoformat(),
"metadataLastUpdated":
"",
}
_prune_dict_null_str(project)
return project
def from_tfs(klass, tfs_project, labor_hours=True):
"""
Creates CodeGovProject object from TFS/VSTS/AzureDevOps Instance
"""
project = klass()
project_web_url = ''
# -- REQUIRED FIELDS --
project['name'] = tfs_project.projectInfo.name
if 'web' in tfs_project.projectInfo._links.additional_properties:
if 'href' in tfs_project.projectInfo._links.additional_properties[
'web']:
# URL Encodes spaces that are in the Project Name for the Project Web URL
project_web_url = requote_uri(
tfs_project.projectInfo._links.additional_properties['web']
['href'])
project['repositoryURL'] = project_web_url
project['homepageURL'] = project_web_url
project['description'] = tfs_project.projectInfo.description
project['vcs'] = 'TFS/AzureDevOps'
project['permissions']['license'] = None
project['tags'] = []
if labor_hours:
logger.debug(
'Sorry labor hour calculation not currently supported.')
# project['laborHours'] = labor_hours_from_url(project['repositoryURL'])
else:
project['laborHours'] = 0
if tfs_project.projectCreateInfo.last_update_time < POLICY_START_DATE:
project['permissions']['usageType'] = 'exemptByPolicyDate'
else:
project['permissions']['usageType'] = 'exemptByAgencyMission'
project['permissions'][
'exemptionText'] = 'This source code resides on a private server and has not been properly evaluated for releaseability.'
project['contact'] = {'email': '', 'URL': project_web_url}
project['date'] = {
'lastModified':
tfs_project.projectLastUpdateInfo.last_update_time.date().
isoformat(),
'created':
tfs_project.projectCreateInfo.last_update_time.date().isoformat(),
'metadataLastUpdated':
''
}
_prune_dict_null_str(project)
return project
def from_stashy(klass, repository, labor_hours=True):
"""
Handles crafting Code.gov Project for Bitbucket Server repositories
"""
# if not isinstance(repository, stashy.repos.Repository):
# raise TypeError('Repository must be a stashy Repository object')
if not isinstance(repository, dict):
raise TypeError("Repository must be a dict")
project = klass()
logger.debug(
"Stashy: project_key=%s repository_slug=%s",
repository["name"],
repository["project"]["key"],
)
# -- REQUIRED FIELDS --
project["name"] = repository["name"]
clone_urls = [clone["href"] for clone in repository["links"]["clone"]]
for url in clone_urls:
# Only rely on SSH Urls for repository urls
if url.startswith("ssh://"):
project["repositoryURL"] = url
break
description = repository["project"].get("description", "")
if description:
project["description"] = "Project description: %s" % description
project["permissions"]["licenses"] = None
web_url = repository["links"]["self"][0]["href"]
public_server = web_url.startswith("https://bitbucket.org")
if repository["public"] and public_server:
project["permissions"]["usageType"] = "openSource"
if labor_hours:
project["laborHours"] = labor_hours_from_url(
project["repositoryURL"])
else:
project["laborHours"] = 0
project["tags"] = ["bitbucket"]
project["contact"]["email"] = ""
project["contact"]["URL"] = repository["links"]["self"][0]["href"]
# -- OPTIONAL FIELDS --
# project['version'] = ''
# project['organization'] = organization.name
# TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
project["status"] = "Development"
project["vcs"] = repository["scmId"]
project["homepageURL"] = repository["links"]["self"][0]["href"]
# project['downloadURL'] =
# project['languages'] =
# project['partners'] = []
# project['relatedCode'] = []
# project['reusedCode'] = []
# date: [object] A date object describing the release. Empty if repo has no commits.
# created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
# lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
if repository.get("created", None):
project["date"] = {
"created": repository["created"],
"lastModified": repository["lastModified"],
}
_prune_dict_null_str(project)
return project
def from_gitlab(klass,
repository,
labor_hours=True,
fetch_languages=False):
"""
Create CodeGovProject object from GitLab Repository
"""
if not isinstance(repository, gitlab.v4.objects.Project):
raise TypeError("Repository must be a gitlab Repository object")
project = klass()
logger.debug(
"GitLab: repository_id=%d path_with_namespace=%s",
repository.id,
repository.path_with_namespace,
)
# -- REQUIRED FIELDS --
project["name"] = repository.name
project["repositoryURL"] = repository.http_url_to_repo
project["description"] = repository.description
# TODO: Update licenses from GitLab API
project["permissions"]["licenses"] = None
web_url = repository.web_url
public_server = web_url.startswith("https://gitlab.com")
if repository.visibility in ("public") and public_server:
project["permissions"]["usageType"] = "openSource"
elif date_parse(repository.created_at) < POLICY_START_DATE:
project["permissions"]["usageType"] = "exemptByPolicyDate"
if labor_hours:
project["laborHours"] = labor_hours_from_url(
project["repositoryURL"])
else:
project["laborHours"] = 0
project["tags"] = ["gitlab"] + repository.tag_list
project["contact"] = {"email": "", "URL": web_url}
# -- OPTIONAL FIELDS --
# project['version'] = ''
project["organization"] = repository.namespace["name"]
# TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
project["status"] = "Development"
project["vcs"] = "git"
project["homepageURL"] = repository.web_url
api_url = repository.manager.gitlab._url
archive_suffix = "/projects/%s/repository/archive" % repository.get_id(
)
project["downloadURL"] = api_url + archive_suffix
# project['languages'] = [l for l, _ in repository.languages()]
if fetch_languages:
project["languages"] = [*repository.languages()]
# project['partners'] = []
# project['relatedCode'] = []
# project['reusedCode'] = []
project["date"] = {
"created":
date_parse(repository.created_at).date().isoformat(),
"lastModified":
date_parse(repository.last_activity_at).date().isoformat(),
"metadataLastUpdated":
"",
}
_prune_dict_null_str(project)
return project
def from_github3(klass, repository, labor_hours=True):
"""
Create CodeGovProject object from github3 Repository object
"""
if not isinstance(repository, github3.repos.repo._Repository):
raise TypeError("Repository must be a github3 Repository object")
logger.info("Processing: %s", repository.full_name)
project = klass()
logger.debug("GitHub3: repository=%s", repository)
# -- REQUIRED FIELDS --
project["name"] = repository.name
project["repositoryURL"] = repository.git_url
project["description"] = repository.description
try:
repo_license = repository.license()
except github3.exceptions.NotFoundError:
logger.debug("no license found for repo=%s", repository)
repo_license = None
if repo_license:
license = repo_license.license
if license:
logger.debug("license spdx=%s; url=%s", license.spdx_id,
license.url)
if license.url is None:
project["permissions"]["licenses"] = [{
"name":
license.spdx_id
}]
else:
project["permissions"]["licenses"] = [{
"URL":
license.url,
"name":
license.spdx_id
}]
else:
project["permissions"]["licenses"] = None
public_server = repository.html_url.startswith("https://github.com")
if not repository.private and public_server:
project["permissions"]["usageType"] = "openSource"
elif date_parse(repository.created_at) < POLICY_START_DATE:
project["permissions"]["usageType"] = "exemptByPolicyDate"
if labor_hours:
project["laborHours"] = labor_hours_from_url(
project["repositoryURL"])
else:
project["laborHours"] = 0
project["tags"] = ["github"]
old_accept = repository.session.headers["Accept"]
repository.session.headers[
"Accept"] = "application/vnd.github.mercy-preview+json"
topics = repository._get(repository.url + "/topics").json()
project["tags"].extend(topics.get("names", []))
repository.session.headers["Accept"] = old_accept
# Hacky way to get an Organization object back with GitHub3.py >= 1.2.0
owner_url = repository.owner.url
owner_api_response = repository._get(owner_url)
organization = repository._json(owner_api_response, 200)
project["contact"]["email"] = organization["email"]
project["contact"]["URL"] = organization["html_url"]
# -- OPTIONAL FIELDS --
# project['version'] = ''
project["organization"] = organization["name"]
# TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
project["status"] = "Development"
project["vcs"] = "git"
project["homepageURL"] = repository.html_url
project["downloadURL"] = repository.downloads_url
project["languages"] = [l for l, _ in repository.languages()]
# project['partners'] = []
# project['relatedCode'] = []
# project['reusedCode'] = []
# date: [object] A date object describing the release.
# created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
# lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
# metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
try:
created_at = repository.created_at.date()
except AttributeError:
created_at = date_parse(repository.created_at).date()
try:
updated_at = repository.updated_at.date()
except AttributeError:
updated_at = date_parse(repository.updated_at).date()
project["date"] = {
"created": created_at.isoformat(),
"lastModified": updated_at.isoformat(),
"metadataLastUpdated": "",
}
_prune_dict_null_str(project)
return project
def from_stashy(klass, repository, labor_hours=True):
"""
Handles crafting Code.gov Project for Bitbucket Server repositories
"""
# if not isinstance(repository, stashy.repos.Repository):
# raise TypeError('Repository must be a stashy Repository object')
if not isinstance(repository, dict):
raise TypeError('Repository must be a dict')
project = klass()
logger.debug(
'Stashy: project_key=%s repository_slug=%s',
repository['name'],
repository['project']['key'],
)
# -- REQUIRED FIELDS --
project['name'] = repository['name']
clone_urls = [clone['href'] for clone in repository['links']['clone']]
for url in clone_urls:
# Only rely on SSH Urls for repository urls
if url.startswith('ssh://'):
project['repositoryURL'] = url
break
description = repository['project'].get('description', '')
if description:
project['description'] = 'Project description: %s' % description
project['permissions']['licenses'] = None
web_url = repository['links']['self'][0]['href']
public_server = web_url.startswith('https://bitbucket.org')
if repository['public'] and public_server:
project['permissions']['usageType'] = 'openSource'
if labor_hours:
project['laborHours'] = labor_hours_from_url(
project['repositoryURL'])
else:
project['laborHours'] = 0
project['tags'] = ['bitbucket']
project['contact']['email'] = ''
project['contact']['URL'] = repository['links']['self'][0]['href']
# -- OPTIONAL FIELDS --
# project['version'] = ''
# project['organization'] = organization.name
# TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
project['status'] = 'Development'
project['vcs'] = repository['scmId']
project['homepageURL'] = repository['links']['self'][0]['href']
# project['downloadURL'] =
# project['languages'] =
# project['partners'] = []
# project['relatedCode'] = []
# project['reusedCode'] = []
# date: [object] A date object describing the release.
# created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
# lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
# metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
# project['date'] = {
# 'created': repository.pushed_at.isoformat(),
# 'lastModified': repository.updated_at.isoformat(),
# 'metadataLastUpdated': '',
# }
_prune_dict_null_str(project)
return project
def from_gitlab(klass, repository, labor_hours=True):
"""
Create CodeGovProject object from GitLab Repository
"""
if not isinstance(repository, gitlab.v4.objects.Project):
raise TypeError('Repository must be a gitlab Repository object')
project = klass()
logger.debug(
'GitLab: repository_id=%d path_with_namespace=%s',
repository.id,
repository.path_with_namespace,
)
# -- REQUIRED FIELDS --
project['name'] = repository.name
project['repositoryURL'] = repository.http_url_to_repo
project['description'] = repository.description
# TODO: Update licenses from GitLab API
project['permissions']['license'] = None
web_url = repository.web_url
public_server = web_url.startswith('https://gitlab.com')
if repository.visibility in ('public') and public_server:
project['permissions']['usageType'] = 'openSource'
elif date_parse(repository.created_at) < POLICY_START_DATE:
project['permissions']['usageType'] = 'exemptByPolicyDate'
if labor_hours:
project['laborHours'] = labor_hours_from_url(
project['repositoryURL'])
else:
project['laborHours'] = 0
project['tags'] = ['gitlab'] + repository.tag_list
project['contact'] = {
'email': '',
'URL': web_url,
}
# -- OPTIONAL FIELDS --
# project['version'] = ''
project['organization'] = repository.namespace['name']
# TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
project['status'] = 'Development'
project['vcs'] = 'git'
project['homepageURL'] = repository.web_url
api_url = repository.manager.gitlab._url
archive_suffix = '/projects/%s/repository/archive' % repository.get_id(
)
project['downloadURL'] = api_url + archive_suffix
# project['languages'] = [l for l, _ in repository.languages()]
# project['partners'] = []
# project['relatedCode'] = []
# project['reusedCode'] = []
project['date'] = {
'created':
date_parse(repository.created_at).date().isoformat(),
'lastModified':
date_parse(repository.last_activity_at).date().isoformat(),
'metadataLastUpdated':
'',
}
_prune_dict_null_str(project)
return project
def from_github3(klass, repository, labor_hours=True):
"""
Create CodeGovProject object from github3 Repository object
"""
if not isinstance(repository, github3.repos.repo.Repository):
raise TypeError('Repository must be a github3 Repository object')
logger.info('Processing: %s', repository.full_name)
project = klass()
logger.debug('GitHub3: repository=%s', repository)
# -- REQUIRED FIELDS --
project['name'] = repository.name
project['repositoryURL'] = repository.git_url
project['description'] = repository.description
repo_license = repository.license()
if repo_license:
license = repo_license.license
if license:
logger.debug('license spdx=%s; url=%s', license['spdx_id'],
license['url'])
if license['url'] is None:
project['permissions']['licenses'] = [{
"name":
license['spdx_id']
}]
else:
project['permissions']['licenses'] = [{
"URL":
license['url'],
"name":
license['spdx_id']
}]
else:
project['permissions']['licenses'] = None
public_server = repository.html_url.startswith('https://github.com')
if not repository.private and public_server:
project['permissions']['usageType'] = 'openSource'
elif date_parse(repository.created_at) < POLICY_START_DATE:
project['permissions']['usageType'] = 'exemptByPolicyDate'
if labor_hours:
project['laborHours'] = labor_hours_from_url(
project['repositoryURL'])
else:
project['laborHours'] = 0
project['tags'] = ['github']
old_accept = repository.session.headers['Accept']
repository.session.headers[
'Accept'] = 'application/vnd.github.mercy-preview+json'
topics = repository._get(repository.url + '/topics').json()
project['tags'].extend(topics['names'])
repository.session.headers['Accept'] = old_accept
organization = repository.owner
project['contact']['email'] = organization.email
project['contact']['URL'] = organization.html_url
# -- OPTIONAL FIELDS --
# project['version'] = ''
project['organization'] = organization.name
# TODO: Currently, can't be an empty string, see: https://github.com/GSA/code-gov-web/issues/370
project['status'] = 'Development'
project['vcs'] = 'git'
project['homepageURL'] = repository.html_url
project['downloadURL'] = repository.download_url
project['languages'] = [l for l, _ in repository.languages()]
# project['partners'] = []
# project['relatedCode'] = []
# project['reusedCode'] = []
# date: [object] A date object describing the release.
# created: [string] The date the release was originally created, in YYYY-MM-DD or ISO 8601 format.
# lastModified: [string] The date the release was modified, in YYYY-MM-DD or ISO 8601 format.
# metadataLastUpdated: [string] The date the metadata of the release was last updated, in YYYY-MM-DD or ISO 8601 format.
project['date'] = {
'created': repository.pushed_at.date().isoformat(),
'lastModified': repository.updated_at.date().isoformat(),
'metadataLastUpdated': '',
}
_prune_dict_null_str(project)
return project
