def main():
args = parse_command_args()
components_with_no_copyrights = []
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url,
token=access_token,
verify=args.no_verify,
timeout=60.0,
retries=4)
project_name, project_version = get_project_version_by_name(
args.project_name, args.version_name)
for component in bd.get_resource("components", project_version):
try:
copyrights = find_copyrights_in_bom_component_origins(component) \
or find_copyrights_in_github_origins(component) \
or find_copyrights_in_previous_component_versions(component)
if copyrights:
print_copyrights(component, copyrights)
else:
components_with_no_copyrights.append(component)
except Exception as e:
logging.debug(
f"Component {component['componentName']} failed with {e}")
if args.report_missing:
report_components_with_no_copyrights(components_with_no_copyrights)
def main():
args = parse_command_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url,
token=access_token,
verify=args.no_verify,
timeout=60.0,
retries=4)
if re.match(".+xlsx?$", args.input_file):
print(f"Processing EXCEL file {args.input_file}")
process_excel_file(args.input_file)
else:
print("Processing as CSV")
process_csv_file(args.input_file)
def main():
upstream_copyright_data_file='copyright_data.txt'
parser = argparse.ArgumentParser("Enumerate BOM componets without copyrigth statements. retrieve copyright statements form upstream channel and/or version")
parser.add_argument("-u", "--base-url", required=True, help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("-t", "--token-file", dest='token_file', required=True, help="containing access token")
parser.add_argument("-nv", "--no-verify", dest='verify', action='store_false', help="disable TLS certificate verification")
parser.add_argument("-o", "--outputdir", dest='outputdir', default='outdir', help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url, token=access_token, verify=args.verify, timeout = 60.0)
outdir=None
if args.outputdir:
outdir = args.outputdir
import os
if not os.path.exists(outdir):
os.mkdir(outdir)
projects = bd.get_resource('projects')
for project in projects:
versions = bd.get_resource('versions',project)
for version in versions:
codelocations = bd.get_resource('codelocations', version)
for codelocation in codelocations:
resources = bd.list_resources(codelocation)
url = resources['scan-data']
filename = url.split('/')[6]
response = bd.session.get(url, stream=True, verify=False)
print (response.status_code)
print (response.ok)
print (url)
if response.ok:
pathname = os.path.join(outdir, filename)
with open(pathname, "wb") as f:
for data in response.iter_content():
f.write(data)
print(f"{filename}, {pathname}")
print(f"Failed to retrieve report {report_id}")
print("Probably not ready yet, waiting 5 seconds then retrying...")
time.sleep(args.sleep_time)
retries -= 1
download_report(location, filename, retries)
else:
raise FailedReportDownload(
f"Failed to retrieve report {report_id} after multiple retries")
# hub = HubInstance()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(base_url=args.bd_url, token=access_token, verify=args.verify)
params = {'q': [f"name:{args.project_name}"]}
projects = [
p for p in bd.get_resource('projects', params=params)
if p['name'] == args.project_name
]
assert len(
projects
) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {'q': [f"versionName:{args.version_name}"]}
versions = [
v for v in bd.get_resource('versions', project, params=params)
if v['versionName'] == args.version_name
def main():
upstream_copyright_data_file = 'copyright_data.txt'
parser = argparse.ArgumentParser(
"Enumerate BOM componets without copyrigth statements. retrieve copyright statements form upstream channel and/or version"
)
parser.add_argument("-u",
"--base-url",
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("-t",
"--token-file",
dest='token_file',
required=True,
help="containing access token")
parser.add_argument("-p",
"--project-name",
dest='project_name',
required=True,
help="Project Name")
parser.add_argument("-v",
"--version-name",
dest='version_name',
required=True,
help="Version Name")
parser.add_argument("-o",
"--output-file",
dest='output_file',
default=upstream_copyright_data_file,
help=f"TEXT-formatted file with data")
parser.add_argument("-jo",
"--json-output-file",
dest='json_output_file',
default=None,
help='JSON formatted file with data')
parser.add_argument("-ukc",
"--use-kb-copyright",
dest='use_kb_copyright',
default=False,
help='Use KB Copyright vs Updated Copyright')
parser.add_argument("-nv",
"--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
global bd
bd = Client(base_url=args.base_url,
token=access_token,
verify=args.verify,
timeout=60.0)
project, version = get_project_version_by_name(args.project_name,
args.version_name)
logging.debug(f"Found {project['name']}:{version['versionName']}")
# retrieve bill of materials and locate entries without copyrights
bom_components_with_no_copyrights = []
for component in bd.get_resource("components", version):
logging.debug(f"Processing component {component['componentName']}")
#if component['componentName'] != 'musl':
# continue
origins = component['origins']
if len(origins) == 0:
bom_components_with_no_copyrights.append(component)
continue
copyrights_count = 0
for origin in origins:
logging.debug(
f"Processing origin {origin['externalNamespace']} {origin['externalId']}"
)
origin_copyrights = [
copyright for copyright in bd.get_resource(
'component-origin-copyrights', origin)
]
copyrights_count += len(origin_copyrights)
if copyrights_count > 0:
break
logging.debug(f"Copyright count = {copyrights_count}")
if copyrights_count == 0:
bom_components_with_no_copyrights.append(component)
logging.debug(
f"Found {len(bom_components_with_no_copyrights)} components with no copyright"
)
output_data = list()
for component in bom_components_with_no_copyrights:
logging.debug(
f"Locating copyrights for component {component['componentName']} {component.get('componentVersionName', None)}"
)
copyrights = find_copyright_for_component(component)
logging.debug(f"Found {len(copyrights)} active records")
output_data.append({'component': component, 'copyrights': copyrights})
logging.debug(f"Writing results into {args.output_file} file")
write_text_output(output_data, args.output_file, args.use_kb_copyright)
if args.json_output_file:
logging.debug(f"Writing results into {args.json_output_file} file")
with open(args.json_output_file, 'w') as f:
json.dump(output_data, f, indent=2)
from blackduck.Authentication import CookieAuth
import argparse
import logging
from pprint import pprint
logging.basicConfig(
level=logging.DEBUG,
format="[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s")
parser = argparse.ArgumentParser()
parser.add_argument("--base-url",
required=True,
help="Hub server URL e.g. https://your.blackduck.url")
parser.add_argument("--username", required=True, help="Hub user")
parser.add_argument("--password", required=True, help="Hub user")
parser.add_argument("--no-verify",
dest='verify',
action='store_false',
help="disable TLS certificate verification")
args = parser.parse_args()
verify = args.verify
base_url = args.base_url
session = HubSession(base_url, timeout=15.0, retries=3, verify=verify)
auth = CookieAuth(session, username=args.username, password=args.password)
bd = Client(base_url=base_url, session=session, auth=auth, verify=verify)
pprint(bd.list_resources())
args = parser.parse_args()
logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s',
stream=sys.stderr,
level=logging.DEBUG)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("blackduck").setLevel(logging.DEBUG)
with open(args.token_file, 'r') as tf:
access_token = tf.readline().strip()
bd = Client(
base_url=args.base_url,
token=access_token,
verify=args.verify,
timeout=args.timeout,
retries=args.retries,
)
params = {'q': [f"name:{args.project_name}"]}
projects = [
p for p in bd.get_resource('projects', params=params)
if p['name'] == args.project_name
]
assert len(
projects
) == 1, f"There should be one, and only one project named {args.project_name}. We found {len(projects)}"
project = projects[0]
params = {'q': [f"versionName:{args.version_name}"]}
session = HubSession(base_url,
timeout=args.timeout,
retries=args.retries,
verify=verify)
# De-tangle the possibilities of specifying credentials
if args.token_file:
tf = open(args.token_file, 'r')
access_token = tf.readline().strip()
auth = BearerAuth(session, access_token)
elif args.username and args.password:
auth = CookieAuth(session, args.username, args.password)
else:
raise SystemError("Authentication credentials not specified")
bd = Client(base_url=base_url, session=session, auth=auth)
projectDict = {} # key:projectId: json
pvDict = {} # key:projectId: json array
for project in sageJson['projects']:
m = re.match(r".*/projects/(.*)", project['url'])
projectId = m.group(1)
projectDict[projectId] = project
pvDict[projectId] = project['versions']
codelocationsDict = {} # key:codelocationId: json
for codelocation in sageJson['scans']:
m = re.match(r".*/codelocations/(.*)", codelocation['url'])
codelocationId = m.group(1)
codelocationsDict[codelocationId] = codelocation
'q': [f"name:{project_name}"]
}
projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name]
assert len(projects) == 1, f"There should be one, and only one project named {project_name}. We found {len(projects)}"
project = projects[0]
params = {
'q': [f"versionName:{version_name}"]
}
versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name]
assert len(versions) == 1, f"There should be one, and only one version named {version_name}. We found {len(versions)}"
version = versions[0]
return project, version
bd = Client(
base_url=args.bd_url,
token=args.user_api_token,
verify=args.verify
)
project, version = get_project_version_by_name(args.project, args.version)
custom_fields = [cf for cf in bd.get_resource("custom-fields", version)]
cfs = list(filter(lambda cf: cf['label'] == args.field_label, custom_fields))
assert len(cfs) == 1, f"We did not find the field labeled {args.field_label} or we found more than one and that shouldn't happen!"
custom_field = cfs[0]
print(f"Custom field {args.field_label} on project-version {args.project}-{args.version} has value(s) {custom_field['values']}")
print(f"Refer to the BD REST API doc for more details on how to interact with the different custom field types, {bd.base_url}/api-doc/public.html#_reading_a_single_project_version_custom_field")