def upload_json(jsonfile): hub = HubInstance() r = hub.upload_scan(jsonfile) if r.status_code == 201: return(True) else: return(False)
def main(argv=None):
'''Command line options.'''
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = '''%s
Created on %s.
Copyright 2019 organization_name. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
try:
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("scanfile", help="specify scan file name")
# Process arguments
args = parser.parse_args()
scanfile = args.scanfile
#
# THE MEAT IS HERE EVERYTHING ELSE IS SALAD
#
#
from blackduck.HubRestApi import HubInstance
hub = HubInstance()
loot = hub.upload_scan(scanfile)
print (loot)
#
#
# MEAT ENDS HERE
#
except KeyboardInterrupt:
### handle keyboard interrupt ###
return 0
except Exception as e:
if DEBUG or TESTRUN:
raise(e)
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help")
return 2
def __init__(self, product, version, cred_file, report_dir):
self.product = product
self.version = version
# Connect to Black Duck
creds = json.load(cred_file)
logger.debug(f"Connecting to Black Duck hub {creds['url']}")
self.hub = HubInstance(creds["url"],
creds["username"],
creds["password"],
insecure=True)
self.report_dir = Path(report_dir)
self.init_reports()
def _query_blackduck():
# pylint: disable=global-statement
global BLACKDUCK_PROJECT_URL
hub = HubInstance()
LOGGER.info("Getting version from blackduck")
version = hub.execute_get(hub.get_urlbase() +
"/api/current-version").json()
LOGGER.info("Version: %s", version)
# Get a list of all projects, this is a privileged call and will fail if we do not have a valid license
LOGGER.info("Get All Projects")
projects = hub.get_projects()
LOGGER.info("Projects: %s", projects)
LOGGER.info("Fetching project %s from blackduck", BLACKDUCK_PROJECT)
project = hub.get_project_by_name(BLACKDUCK_PROJECT)
LOGGER.info("Fetching project version %s from blackduck",
BLACKDUCK_PROJECT_VERSION)
version = hub.get_version_by_name(project, BLACKDUCK_PROJECT_VERSION)
LOGGER.info("Getting version components from blackduck")
bom_components = hub.get_version_components(version)
components = [
Component.parse(hub, comp) for comp in bom_components["items"]
if comp['ignored'] is False
]
BLACKDUCK_PROJECT_URL = version["_meta"]["href"]
return components
def delete_codelocations(delete_mapped, delete_complete_only, before_date, dry_run):
hub = HubInstance()
message = '''
Processing scans with following parameters
deleting mapped scans: {}
deleting complete scans only: {}
before date specified as: {}
Dry run: {}
'''
if before_date:
beforedate = datetime.fromisoformat(before_date)
print(message.format(delete_mapped, delete_complete_only, before_date, dry_run))
code_locations = hub.get_codelocations(500, not delete_mapped).get('items', [])
for c in code_locations:
scan_summaries = hub.get_codelocation_scan_summaries(code_location_obj = c).get('items', [])
name = c['name']
created = datetime.fromisoformat(scan_summaries[0]['createdAt'].replace("Z", ""))
status = scan_summaries[0]['status']
print ("processing {} created on {} status {}".format(name, created, status))
dateChecks = False
if before_date:
if created < beforedate:
dateChecks = True
else:
dateChecks = True
completeChecks = False
if delete_complete_only:
if status == 'COMPLETE':
completeChecks = True
else:
completeChecks = True
if dateChecks and completeChecks:
print ("executing delete request")
print (not dry_run)
if not dry_run:
print (hub.execute_delete(c['_meta']['href']))
else:
print("Dry run, no deletion")
else:
print('will not delete ')
def test_hub_instance_with_write_config_false(requests_mock):
requests_mock.post(
"https://my-hub-host/j_spring_security_check",
headers={
"Set-Cookie":
'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.format(
invalid_bearer_token)
})
requests_mock.get("{}/api/current-version".format(fake_hub_host),
json={
"version": "2018.11.0",
"_meta": {
"allow": ["GET"],
"href":
"{}/api/current-version".format(fake_hub_host)
}
})
with patch.object(HubInstance, "write_config") as mock_write_config:
hub = HubInstance(fake_hub_host,
"a_username",
"a_password",
write_config_flag=False)
assert not mock_write_config.called
def mock_hub_instance(requests_mock):
requests_mock.post("https://my-hub-host/api/tokens/authenticate",
content=json.dumps({
'bearerToken': invalid_bearer_token
}).encode('utf-8'),
headers={
'X-CSRF-TOKEN': invalid_csrf_token,
'Content-Type': 'application/json'
})
requests_mock.get("{}/api/current-version".format(fake_hub_host),
json={
"version": hub_version,
"_meta": {
"allow": ["GET"],
"href":
"{}/api/current-version".format(fake_hub_host)
}
})
yield HubInstance(fake_hub_host, api_token=made_up_api_token)
try:
os.remove(f_name)
print("Removed {}".format(f_name))
except:
print("Failed to remove {}".format(f_name))
def main(argv=None):
if argv is None:
argv = sys.argv
else:
argv.extend(sys.argv)
parser = ArgumentParser()
parser.add_argument(
'imagespec',
help="Container image tag, e.g. repository/imagename:version")
parser.add_argument('--cleanup',
default=False,
help="Delete project hierarchy only. Do not scan")
parser.add_argument(
'--rescan-layer',
default=0,
type=int,
help="Rescan specific layer in case of failure, 0 - scan as usual")
args = parser.parse_args()
print(args)
hub = HubInstance()
if args.cleanup:
clean_container_project(args.imagespec)
else:
if args.rescan_layer == 0:
clean_container_project(args.imagespec)
scan_container_image(args.imagespec)
else:
scan_container_image(args.imagespec, args.rescan_layer)
def scan_container_image(imagespec,
grouping=None,
base_image=None,
dockerfile=None,
project_name=None,
project_version=None,
detect_options=None):
hub = HubInstance()
scanner = ContainerImageScanner(hub,
imagespec,
grouping=grouping,
base_image=base_image,
dockerfile=dockerfile,
detect_options=detect_options)
if project_name:
scanner.project_name = project_name
if project_version:
scanner.project_version = project_version
if not grouping:
if not base_image and not dockerfile:
scanner.grouping = '1024:everything'
else:
scanner.base_layers = scanner.get_base_layers()
scanner.prepare_container_image()
scanner.process_container_image()
scanner.submit_layer_scans()
def main(argv=None): # IGNORE:C0111
'''Command line options.'''
if argv is None:
argv = sys.argv
else:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = '''%s
Created by user_name on %s.
Copyright 2020 organization_name. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_shortdesc, str(__date__))
try:
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument(dest="input_path", help="path to folder to store exported data")
# Process arguments
args = parser.parse_args()
input_path = args.input_path
workdir = os.path.abspath(input_path)
metadata_path = os.path.join(workdir, "metadata.json")
print("Reading metadata from {} ".format(metadata_path))
metadata = read_metadata_file(metadata_path)
hub=HubInstance()
create_projects(hub, metadata)
upload_scan_data(hub, workdir)
return 0
except Exception as e:
import traceback
traceback.print_stack()
if DEBUG or TESTRUN:
raise(e)
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help")
return 2
def initialize(cls):
if not cls.hub:
logging.debug("setting hub")
cls.hub = HubInstance()
if not cls.jobs:
logging.debug("retrieving jobs")
cls.jobs = cls.hub.get_jobs().get('items', [])
def initialize(cls):
if not cls.hub:
logging.debug("setting hub")
cls.hub = HubInstance()
if not cls.jobs:
logging.debug("retrieving jobs")
cls.jobs = cls._get_all_jobs()
def scan_container_image(imagespec):
hub = HubInstance()
scanner = ContainerImageScanner(hub, imagespec)
scanner.prepare_container_image()
scanner.process_container_image()
scanner.generate_project_structures()
scanner.submit_layer_scans()
def remove_empty_versions(project_id, cleanup=False):
hub = HubInstance()
project = hub.get_project_by_id(project_id, limit=100)
print(project['name'])
versions = hub.get_project_versions(project, limit=200)
print("\t versions found %s" % versions['totalCount'])
versionlist = versions['items']
for index in range(len(versionlist) - 1):
while True:
count = 0
try:
va = versionlist[index]
components = hub.get_version_components(va, limit=1)
totalCount = components['totalCount']
print("Vesion {} has {} components".format(
va['versionName'], totalCount))
if cleanup and totalCount == 0:
print("removing {}".format(va['_meta']['href']))
hub.execute_delete(va['_meta']['href'])
break
except Exception:
count += 1
print("Oops! Attempt number {} failed".format(count))
if count > 3:
break
continue
else:
break
def delete_empty_projects(before=None):
hub = HubInstance()
logging.basicConfig(level=logging.DEBUG)
projects = hub.get_projects(limit=500)
print(projects)
if before:
aDate = datetime.strptime(before, '%Y-%m-%d')
else:
aDate = datetime.now() + timedelta(days=1)
for project in projects['items']:
name = project['name']
# uuid = project['_meta']['href'].split('/')[5]
updatedAt = project['updatedAt']
uDate = datetime.strptime(updatedAt, '%Y-%m-%dT%H:%M:%S.%fZ')
componentCount = 0
if uDate < aDate:
print("Processing {} ".format(name))
versions = hub.get_project_versions(project)
for version in versions['items']:
components = hub.get_version_components(version, limit=1)
componentCount += components['totalCount']
if componentCount == 0:
print("Project {} has {} components".format(
name, componentCount))
print("attempting to delete")
hub.delete_project_by_name(name, save_scans=True)
def mock_hub_instance(requests_mock):
requests_mock.post(
"https://my-hub-host/j_spring_security_check",
headers={
"Set-Cookie":
'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.format(
invalid_bearer_token)
})
yield HubInstance(fake_hub_host, "a_username", "a_password")
def test_hub_instance_with_write_config_false(requests_mock):
requests_mock.post(
"https://my-hub-host/j_spring_security_check",
headers={"Set-Cookie": 'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.format(invalid_bearer_token)}
)
with patch.object(HubInstance, "write_config") as mock_write_config:
hub = HubInstance(fake_hub_host, "a_username", "a_password", write_config_flag=False)
assert not mock_write_config.called
def process_blackduck_report(reportJson):
riskUrl = ''
hub = HubInstance(BLACKDUCK_URL,
api_token=BLACKDUCK_API_KEY,
insecure=True)
#'docker_web_app'
projects = hub.get_project_by_name(BLACKDUCK_PROJECT_NAME)
projectVersions = hub.get_project_versions(project=projects)
for item in projectVersions['items'][0]['_meta']['links']:
if item['rel'] == 'riskProfile':
riskUrl = item['href']
riskData = hub.execute_get(url=riskUrl)
vulnerableData = riskData.json()['categories']['VULNERABILITY']
reportJson['customParameters']['blackduckReport'] = vulnerableData
return reportJson
def test_hub_instance_with_write_config(requests_mock):
requests_mock.post(
"https://my-hub-host/j_spring_security_check",
headers={"Set-Cookie": 'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.format(invalid_bearer_token)}
)
with patch("builtins.open", new_callable=mock_open()) as m:
with patch('json.dump') as m_json:
hub = HubInstance(fake_hub_host, "a_username", "a_password")
m.assert_called_with('.restconfig.json', 'w')
assert m_json.called
def _query_blackduck():
# pylint: disable=global-statement
global BLACKDUCK_PROJECT_URL
hub = HubInstance()
LOGGER.info("Fetching project %s from blackduck", BLACKDUCK_PROJECT)
project = hub.get_project_by_name(BLACKDUCK_PROJECT)
LOGGER.info("Fetching project version %s from blackduck", BLACKDUCK_PROJECT_VERSION)
version = hub.get_version_by_name(project, BLACKDUCK_PROJECT_VERSION)
LOGGER.info("Getting version components from blackduck")
bom_components = hub.get_version_components(version)
components = [Component.parse(hub, comp) for comp in bom_components["items"]]
BLACKDUCK_PROJECT_URL = version["_meta"]["href"]
return components
def mock_hub_instance_using_api_token(requests_mock):
requests_mock.post("https://my-hub-host/api/tokens/authenticate",
content=json.dumps({
'bearerToken': invalid_bearer_token
}).encode('utf-8'),
headers={
'X-CSRF-TOKEN': invalid_csrf_token,
'Content-Type': 'application/json'
})
yield HubInstance(fake_hub_host, api_token=made_up_api_token)
def scan_container_image(imagespec, layer_number=0):
hub = HubInstance()
scanner = ContainerImageScanner(hub, imagespec)
scanner.prepare_container_image()
scanner.process_container_image()
if layer_number == 0:
scanner.generate_project_structures()
scanner.submit_layer_scans()
else:
scanner.generate_single_layer_project_structure(layer_number)
scanner.submit_single_layer_scan(int(layer_number))
def __init__(self, product, version, bld_num, cred_file, output_dir):
"""
Connects to Hub and initializes Project and Version
"""
self.product_name = product
self.version_name = version
self.prefix = f"{product}-{version}-{bld_num}"
self.output_dir = Path(output_dir).resolve()
self.tmp_zip = self.output_dir / "tmp.zip"
# Connect to Black Duck
creds = json.load(cred_file)
logger.info(f"Connecting to Black Duck hub {creds['url']}")
self.hub = HubInstance(creds["url"], creds["username"], creds["password"], insecure=True)
self.product = self.hub.get_project_by_name(product)
if not self.product:
raise Exception(f"Unknown product {product}")
self.version = self.hub.get_version_by_name(self.product, version)
if not self.version:
raise Exception(f"Unknown version {version} for product {product}")
def scan_container_image_with_dockerfile(imagespec, dockerfile, base_image,
omit_base_layers):
hub = HubInstance()
scanner = ContainerImageScanner(hub,
imagespec,
dockerfile=dockerfile,
base_image=base_image,
omit_base_layers=omit_base_layers)
base_layers = scanner.get_base_layers()
print(base_layers)
scanner.prepare_container_image()
scanner.process_container_image()
scanner.generate_project_structures(base_layers)
scanner.submit_layer_scans()
def scan_container_image(imagespec,
grouping=None,
base_image=None,
dockerfile=None):
hub = HubInstance()
scanner = ContainerImageScanner(hub,
imagespec,
grouping=grouping,
base_image=base_image,
dockerfile=dockerfile)
if not grouping:
scanner.base_layers = scanner.get_base_layers()
scanner.prepare_container_image()
scanner.process_container_image()
scanner.submit_layer_scans()
def mock_hub_instance(requests_mock):
requests_mock.post(
"{}/j_spring_security_check".format(fake_hub_host),
headers={
"Set-Cookie":
'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.format(
invalid_bearer_token)
})
requests_mock.get("{}/api/current-version".format(fake_hub_host),
json={
"version": "2018.11.0",
"_meta": {
"allow": ["GET"],
"href":
"{}/api/current-version".format(fake_hub_host)
}
})
yield HubInstance(fake_hub_host, "a_username", "a_password")
def _create_hub_instance(hub_version=default_hub_version):
requests_mock.post(
"{}/j_spring_security_check".format(fake_hub_host),
headers={
"Set-Cookie":
'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.
format(fake_bearer_token)
})
requests_mock.get(
"{}/api/current-version".format(fake_hub_host),
json={
"version": hub_version,
"_meta": {
"allow": ["GET"],
"href": "{}/api/current-version".format(fake_hub_host)
}
})
return HubInstance(fake_hub_host, "a_username", "a_password")
def mock_hub_instance_using_api_token(requests_mock):
requests_mock.post("https://my-hub-host/api/tokens/authenticate",
content=json.dumps({
'bearerToken': invalid_bearer_token
}).encode('utf-8'),
headers={
'X-CSRF-TOKEN': invalid_csrf_token,
'Content-Type': 'application/json'
})
requests_mock.get("{}/api/current-version".format(fake_hub_host),
json={
"version": "2018.11.0",
"_meta": {
"allow": ["GET"],
"href":
"{}/api/current-version".format(fake_hub_host)
}
})
yield HubInstance(fake_hub_host, api_token=made_up_api_token)
def test_get_major_version(requests_mock):
requests_mock.post(
"{}/j_spring_security_check".format(fake_hub_host),
headers={
"Set-Cookie":
'AUTHORIZATION_BEARER={}; Path=/; secure; Secure; HttpOnly'.format(
invalid_bearer_token)
})
for version in ["2018.11.0", "5.0.2", "4.8.3", "3.7.2"]:
expected = version.split(".")[0]
requests_mock.get(
"{}/api/current-version".format(fake_hub_host),
json={
"version": "{}".format(version),
"_meta": {
"allow": ["GET"],
"href": "{}/api/current-version".format(fake_hub_host)
}
})
hub = HubInstance(fake_hub_host, "a_username", "a_password")
assert hub.bd_major_version == expected
def main(argv=None):
if argv is None:
argv = sys.argv
else:
argv.extend(sys.argv)
parser = ArgumentParser()
parser.add_argument(
'imagespec',
help="Container image tag, e.g. repository/imagename:version")
parser.add_argument('--cleanup',
default=False,
help="Delete project hierarchy only. Do not scan")
args = parser.parse_args()
hub = HubInstance()
clean_container_project(args.imagespec)
if not args.cleanup:
scan_container_image(args.imagespec)
