def main():
args = parse_command_args()
components_with_no_copyrights = []
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url,
token=access_token,
verify=args.no_verify,
timeout=60.0,
retries=4)
project_name, project_version = get_project_version_by_name(
args.project_name, args.version_name)
for component in bd.get_resource("components", project_version):
try:
copyrights = find_copyrights_in_bom_component_origins(component) \
or find_copyrights_in_github_origins(component) \
or find_copyrights_in_previous_component_versions(component)
if copyrights:
print_copyrights(component, copyrights)
else:
components_with_no_copyrights.append(component)
except Exception as e:
logging.debug(
f"Component {component['componentName']} failed with {e}")
if args.report_missing:
report_components_with_no_copyrights(components_with_no_copyrights)
def cb_configserver(button, bdserver, apikey, usecerts):
global bd
if button is None:
raise dash.exceptions.PreventUpdate
if usecerts == 'INSECURE':
verify = False
else:
verify = True
bd = Client(
token=apikey,
base_url=bdserver,
timeout=300,
verify=verify # TLS certificate verification
)
try:
bd.list_resources()
except Exception as exc:
toast = make_connect_toast('Unable to eonnect - {}'.format(str(exc)))
return True, None, '', '', '', False, toast
projdf = projs.get_project_data(bd)
projlabel = "Projects (" + str(len(projdf.index)) + ")"
return False, projdf.to_dict(
'records'), projlabel, bdserver, apikey, verify, ''
def main():
args = parse_command_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url,
token=access_token,
verify=args.no_verify,
timeout=60.0,
retries=4)
if re.match(".+xlsx?$", args.input_file):
print(f"Processing EXCEL file {args.input_file}")
process_excel_file(args.input_file)
else:
print("Processing as CSV")
process_csv_file(args.input_file)
def main():
upstream_copyright_data_file='copyright_data.txt'
parser = argparse.ArgumentParser("Enumerate BOM componets without copyrigth statements. retrieve copyright statements form upstream channel and/or version")
parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("-t", "--token-file", dest='token_file', required=True, help="containing access token")
parser.add_argument("-nv", "--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
parser.add_argument("-o", "--outputdir", dest='outputdir', default='outdir', help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify, timeout = 60.0)
outdir=None
if args.outputdir:
outdir = args.outputdir
import os
if not os.path.exists(outdir):
os.mkdir(outdir)
projects = bd.get_resource('projects')
for project in projects:
versions = bd.get_resource('versions',project)
for version in versions:
codelocations = bd.get_resource('codelocations', version)
for codelocation in codelocations:
resources = bd.list_resources(codelocation)
url = resources['scan-data']
filename = url.split('/')[6]
response = bd.session.get(url, stream=True, verify=False)
print (response.status_code)
print (response.ok)
print (url)
if response.ok:
pathname = os.path.join(outdir, filename)
with open(pathname, "wb") as f:
for data in response.iter_content():
f.write(data)
print(f"{filename}, {pathname}")
required=True,
help="Project that contains the BOM components")
parser.add_argument("--version",
dest='version_name',
required=True,
help="Version that contains the BOM components")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
params = {'q': [f"name:{args.project_name}"]}
projects = [
p for p in bd.get_resource('projects', params=params)
if p['name'] == args.project_name
]
assert len(
projects
) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {'q': [f"versionName:{args.version_name}"]}
versions = [
v for v in bd.get_resource('versions', project, params=params)
if v['versionName'] == args.version_name
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
stream=sys.stderr,
level=logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("blackduck").setLevel(logging.DEBUG)
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(
base_url=args.base_url,
token=access_token,
verify=args.verify,
timeout=args.timeout,
retries=args.retries,
)
params = {'q': [f"name:{args.project_name}"]}
projects = [
p for p in bd.get_resource('projects', params=params)
if p['name'] == args.project_name
]
assert len(
projects
) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {'q': [f"versionName:{args.version_name}"]}
from pprint import pprint
logging.basicConfig(
level=logging.DEBUG,
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s"
)
parser = argparse.ArgumentParser("Get a specific component and list its vulnerabilities")
parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token")
parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
params = {
'q': ["maven:commons-beanutils:commons-beanutils:1.9.3"]
}
search_results = bd.get_items("/api/components", params=params)
for result in search_results:
pprint(result)
print(f"{result['componentName']} {result['versionName']}")
url = result['version']
component_version = bd.get_json(url)
for vulnerability in bd.get_resource('vulnerabilities', component_version):
print(vulnerability['name'])
required=True,
help="containing access token")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
parser.add_argument(
"cpe_id",
help=
"Provide a CPE (2.2 or 2.3 xml format) ID - e.g. \"cpe:2.3:a:apache:log4j:2.11.1:-:*:*:*:*:*:*\" To get a complete dictionary of CPE IDs go to the NIST site, https://nvd.nist.gov/products/cpe"
)
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
stream=sys.stderr,
level=logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("blackduck").setLevel(logging.WARNING)
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
cpes = [cpe for cpe in bd.get_items(f"/api/cpes?q={args.cpe_id}")]
if cpes:
for cpe in cpes:
cpe['cpe-origins'] = [o for o in bd.get_resource("cpe-origins", cpe)]
cpe['cpe-versions'] = [v for v in bd.get_resource("cpe-versions", cpe)]
print(json.dumps(cpes))
args = parser.parse_args()
verify = False # TLS certificate verification
session = HubSession(args.base_url, timeout=15.0, retries=3, verify=verify)
# De-tangle the possibilities of specifying credentials
if args.token_file:
tf = open(args.token_file, 'r')
access_token = tf.readline().strip()
auth = BearerAuth(session, access_token)
elif args.username and args.password:
auth = CookieAuth(session, args.username, args.password)
else:
raise SystemError("Authentication credentials not specified")
bd = Client(base_url=args.base_url, session=session, auth=auth, verify=verify)
if args.mode not in ['list', 'delete']:
print("Error: must specify --mode to be one of: list or delete")
sys.exit(-1)
input_file = open(args.csv_file_input, 'r')
reader = csv.DictReader(input_file)
logging.info("csv fieldnames:")
logging.info(reader.fieldnames)
check_for_columns([
'projectId',
'project',
'versionId',
logging.basicConfig(
level=logging.DEBUG,
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s")
parser = argparse.ArgumentParser("Get a single vulnerability by id")
parser.add_argument("--base-url",
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--token-file",
dest='token_file',
required=True,
help="containing access token")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
parser.add_argument(
"--vulnerability",
help="vulnerability id e.g. CVE-2016-4009 or BDSA-2014-0129")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
vulnerability = bd.get_json(f"/api/vulnerabilities/{args.vulnerability}")
print(f"Details about {vulnerability['name']}")
pprint(vulnerability)
level=logging.INFO,
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s"
)
parser = argparse.ArgumentParser("Get the BOM components for a given project-version and the license details for each BOM component")
parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token")
parser.add_argument("--project", dest='project_name', required=True, help="Project that contains the BOM components")
parser.add_argument("--version", dest='version_name', required=True, help="Version that contains the BOM components")
parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
params = {
'q': [f"name:{args.project_name}"]
}
projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name]
assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {
'q': [f"versionName:{args.version_name}"]
}
versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name]
assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}"
version = versions[0]
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--token-file",
dest='token_file',
required=True,
help="containing access token")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
def iterate_sub_resources(parent, breadcrumb=""):
try:
resources = bd.list_resources(parent)
except TypeError:
return
for res in resources:
print(f"{breadcrumb}->{res}", end='')
if res in [
'scan-data', # 200; but .bdio output is not json
'apiDocumentation', # 200; but output is just a huge amount of html
'detectUri', # 204; no content
]:
'q': [f"name:{project_name}"]
}
projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name]
assert len(projects) == 1, f"There should be one, and only one project named {project_name}. We found {len(projects)}"
project = projects[0]
params = {
'q': [f"versionName:{version_name}"]
}
versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name]
assert len(versions) == 1, f"There should be one, and only one version named {version_name}. We found {len(versions)}"
version = versions[0]
return project, version
bd = Client(
base_url=args.bd_url,
token=args.user_api_token,
verify=args.verify
)
project, version = get_project_version_by_name(args.project, args.version)
custom_fields = [cf for cf in bd.get_resource("custom-fields", version)]
cfs = list(filter(lambda cf: cf['label'] == args.field_label, custom_fields))
assert len(cfs) == 1, f"We did not find the field labeled {args.field_label} or we found more than one and that shouldn't happen!"
custom_field = cfs[0]
print(f"Custom field {args.field_label} on project-version {args.project}-{args.version} has value(s) {custom_field['values']}")
print(f"Refer to the BD REST API doc for more details on how to interact with the different custom field types, {bd.base_url}/api-doc/public.html#_reading_a_single_project_version_custom_field")
print(f"Successfully downloaded zip file to {filename} for report {report_id}")
else:
print(f"Failed to retrieve report {report_id}")
print("Probably not ready yet, waiting 5 seconds then retrying...")
time.sleep(args.sleep_time)
retries -= 1
download_report(location, filename, retries)
else:
raise FailedReportDownload(f"Failed to retrieve report {report_id} after multiple retries")
# hub = HubInstance()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify)
params = {
'q': [f"name:{args.project_name}"]
}
projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == args.project_name]
assert len(projects) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {
'q': [f"versionName:{args.version_name}"]
}
versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == args.version_name]
assert len(versions) == 1, f"There should be one, and only one version named {args.version_name}. We found {len(versions)}"
version = versions[0]
from blackduck.Authentication import CookieAuth
import argparse
import logging
from pprint import pprint
logging.basicConfig(
level=logging.DEBUG,
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s")
parser = argparse.ArgumentParser()
parser.add_argument("--base-url",
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--username", required=True, help="Hub user")
parser.add_argument("--password", required=True, help="Hub user")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
verify = args.verify
base_url = args.base_url
session = HubSession(base_url, timeout=15.0, retries=3, verify=verify)
auth = CookieAuth(session, username=args.username, password=args.password)
bd = Client(base_url=base_url, session=session, auth=auth, verify=verify)
pprint(bd.list_resources())
session = HubSession(base_url,
timeout=args.timeout,
retries=args.retries,
verify=verify)
# De-tangle the possibilities of specifying credentials
if args.token_file:
tf = open(args.token_file, 'r')
access_token = tf.readline().strip()
auth = BearerAuth(session, access_token)
elif args.username and args.password:
auth = CookieAuth(session, args.username, args.password)
else:
raise SystemError("Authentication credentials not specified")
bd = Client(base_url=base_url, session=session, auth=auth)
projectDict = {} # key:projectId: json
pvDict = {} # key:projectId: json array
for project in sageJson['projects']:
m = re.match(r".*/projects/(.*)", project['url'])
projectId = m.group(1)
projectDict[projectId] = project
pvDict[projectId] = project['versions']
codelocationsDict = {} # key:codelocationId: json
for codelocation in sageJson['scans']:
m = re.match(r".*/codelocations/(.*)", codelocation['url'])
codelocationId = m.group(1)
codelocationsDict[codelocationId] = codelocation
f"Set the base name for the output file (default: {DEFAULT_OUTPUT_BASE}. The format will determine the file extension (e.g. format=JSON means file will end with .json)"
)
# parser.add_argument("-u", "--usage-details", dest="usage_details", action='store_true', help="Use this option to retrieve license usage details which will show what project-versions and components are using the license")
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
stream=sys.stderr,
level=logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("blackduck").setLevel(logging.WARNING)
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
# WARNING:
# This uses an internal, un-supported API endpoint (see below)
# and therefore could break in future versions of Black Duck.
# This script was tested with BD v2021.8.2.
#
# An ER has been filed in Synopsys Jira ticket HUB-26144 to make a
# in-use filter part of a public endpoint for retrieving licenses
inuse_licenses_url = f"{bd.base_url}/api/internal/composite/licenses?filter=inUse:true"
headers = {'Accept': 'application/vnd.blackducksoftware.internal-1+json'}
inuse_licenses_d = {
l['name']: l
for l in bd.get_items(inuse_licenses_url, headers=headers)
def main():
upstream_copyright_data_file = 'copyright_data.txt'
parser = argparse.ArgumentParser(
"Enumerate BOM componets without copyrigth statements. retrieve copyright statements form upstream channel and/or version"
)
parser.add_argument("-u",
"--base-url",
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("-t",
"--token-file",
dest='token_file',
required=True,
help="containing access token")
parser.add_argument("-p",
"--project-name",
dest='project_name',
required=True,
help="Project Name")
parser.add_argument("-v",
"--version-name",
dest='version_name',
required=True,
help="Version Name")
parser.add_argument("-o",
"--output-file",
dest='output_file',
default=upstream_copyright_data_file,
help=f"TEXT-formatted file with data")
parser.add_argument("-jo",
"--json-output-file",
dest='json_output_file',
default=None,
help='JSON formatted file with data')
parser.add_argument("-ukc",
"--use-kb-copyright",
dest='use_kb_copyright',
default=False,
help='Use KB Copyright vs Updated Copyright')
parser.add_argument("-nv",
"--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url,
token=access_token,
verify=args.verify,
timeout=60.0)
project, version = get_project_version_by_name(args.project_name,
args.version_name)
logging.debug(f"Found {project['name']}:{version['versionName']}")
# retrieve bill of materials and locate entries without copyrights
bom_components_with_no_copyrights = []
for component in bd.get_resource("components", version):
logging.debug(f"Processing component {component['componentName']}")
#if component['componentName'] != 'musl':
# continue
origins = component['origins']
if len(origins) == 0:
bom_components_with_no_copyrights.append(component)
continue
copyrights_count = 0
for origin in origins:
logging.debug(
f"Processing origin {origin['externalNamespace']} {origin['externalId']}"
)
origin_copyrights = [
copyright for copyright in bd.get_resource(
'component-origin-copyrights', origin)
]
copyrights_count += len(origin_copyrights)
if copyrights_count > 0:
break
logging.debug(f"Copyright count = {copyrights_count}")
if copyrights_count == 0:
bom_components_with_no_copyrights.append(component)
logging.debug(
f"Found {len(bom_components_with_no_copyrights)} components with no copyright"
)
output_data = list()
for component in bom_components_with_no_copyrights:
logging.debug(
f"Locating copyrights for component {component['componentName']} {component.get('componentVersionName', None)}"
)
copyrights = find_copyright_for_component(component)
logging.debug(f"Found {len(copyrights)} active records")
output_data.append({'component': component, 'copyrights': copyrights})
logging.debug(f"Writing results into {args.output_file} file")
write_text_output(output_data, args.output_file, args.use_kb_copyright)
if args.json_output_file:
logging.debug(f"Writing results into {args.json_output_file} file")
with open(args.json_output_file, 'w') as f:
json.dump(output_data, f, indent=2)
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--token-file",
dest='token_file',
required=True,
help="containing access token")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify)
for project in bd.get_resource('projects'):
print(f"Project: {project['name']}")
print("Project list_resources():")
pprint(bd.list_resources(project))
for version in bd.get_resource('versions', project):
print(f"Version: {version['versionName']}")
for bom_component in bd.get_resource('components', version):
print(
f"BOM component: {bom_component['componentName']}:{bom_component['componentVersionName']}"
)
# print(f"Version list_resources():")
# pprint(bd.list_resources(version))
# print("Exiting after printing first project and version")
# quit(0)
print(f"Failed to retrieve report {report_id}")
print("Probably not ready yet, waiting 5 seconds then retrying...")
time.sleep(args.sleep_time)
retries -= 1
download_report(location, filename, retries)
else:
raise FailedReportDownload(
f"Failed to retrieve report {report_id} after multiple retries")
# hub = HubInstance()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify)
params = {'q': [f"name:{args.project_name}"]}
projects = [
p for p in bd.get_resource('projects', params=params)
if p['name'] == args.project_name
]
assert len(
projects
) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {'q': [f"versionName:{args.version_name}"]}
versions = [
v for v in bd.get_resource('versions', project, params=params)
if v['versionName'] == args.version_name
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s"
)
parser = argparse.ArgumentParser("Create, read, update, and delete a project")
parser.add_argument("--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--token-file", dest='token_file', required=True, help="containing access token")
parser.add_argument("--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
parser.add_argument("--project", required=True, help="Project name")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(
base_url=args.base_url,
token=access_token,
verify=args.verify
)
project_name = args.project
# POST
project_data = {
'name': project_name,
'description': "some description",
'projectLevelAdjustments': True,
}
try:
r = bd.session.post("/api/projects", json=project_data)
r.raise_for_status()