def test_read_with_create_default_keychain(self):
logger.info(self.getTestHeader('read keychain with create default if missing'))
try:
keychain_file = ospj(self.test_config_dir, ".bdbag", 'keychain.json')
keychain.read_keychain(keychain_file=keychain_file)
except Exception as e:
self.fail(get_typed_exception(e))
def test_read_with_create_default_keychain_location(self):
logger.info(self.getTestHeader('test read keychain with create default location'))
try:
default_keychain_path = ospj(self.test_config_dir, ".bdbag")
default_keychain_file = ospj(default_keychain_path, 'keychain.json')
patched_default_config = mock.patch.multiple(
"bdbag.fetch.auth.keychain",
DEFAULT_KEYCHAIN_FILE=default_keychain_file)
patched_default_config.start()
keychain.read_keychain(keychain_file=default_keychain_file)
patched_default_config.stop()
except Exception as e:
self.fail(get_typed_exception(e))
def test_update_keychain_add_multi(self):
logger.info(self.getTestHeader('test update keychain add multi'))
keychain_file = ospj(self.test_config_dir, 'test-keychain-8.json')
added_entries = [{
"uri": "https://foo.bar.com/",
"auth_type": "http-basic",
"auth_params": {
"auth_method": "get",
"username": "******",
"password": "******"
}
}, {
"uri": "https://foo.bar.com/",
"auth_type": "bearer-token",
"auth_params": {
"token": "bar",
"allow_redirects_with_token": "True",
"additional_request_headers": {
"X-Requested-With": "XMLHttpRequest"
}
}
}]
try:
keychain = read_keychain(keychain_file, create_default=False)
entries = get_auth_entries("https://foo.bar.com/", keychain)
self.assertFalse(entries)
updated_keychain = update_keychain(added_entries,
keychain_file=keychain_file)
logger.info("Updated keychain: %s" % json.dumps(updated_keychain))
entries = get_auth_entries("https://foo.bar.com/",
updated_keychain)
self.assertTrue(len(entries) == 2)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_update_keychain_del_by_tag(self):
logger.info(self.getTestHeader('test update keychain del by tag'))
keychain_file = ospj(self.test_config_dir, 'test-keychain-8.json')
deleted_entries = {"tag": "unit test"}
try:
keychain = read_keychain(keychain_file, create_default=False)
entries = get_auth_entries("https://raw.githubusercontent.com/",
keychain)
self.assertTrue(entries)
updated_keychain = update_keychain(deleted_entries,
keychain_file=keychain_file,
delete=True)
logger.info("Updated keychain: %s" % json.dumps(updated_keychain))
entries = get_auth_entries("https://raw.githubusercontent.com/",
updated_keychain)
self.assertFalse(entries)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_update_keychain_del_single(self):
logger.info(self.getTestHeader('test update keychain del single'))
keychain_file = ospj(self.test_config_dir, 'test-keychain-8.json')
deleted_entry = {
"uri": "ftp://ftp.nist.gov/",
"auth_type": "ftp-basic"
}
try:
keychain = read_keychain(keychain_file, create_default=False)
entries = get_auth_entries("ftp://ftp.nist.gov/", keychain)
self.assertTrue(len(entries) == 1)
updated_keychain = update_keychain(deleted_entry,
keychain_file=keychain_file,
delete=True)
logger.info("Updated keychain: %s" % json.dumps(updated_keychain))
entries = get_auth_entries("ftp://ftp.nist.gov/", updated_keychain)
self.assertFalse(entries)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_update_keychain_invalid_params(self):
logger.info(self.getTestHeader('test update keychain invalid params'))
keychain_file = ospj(self.test_config_dir, 'test-keychain-8.json')
deleted_entries = [{
"uri": "https://raw.githubusercontent.com/",
}, {
"auth_type": "bearer-token"
}, {
"uri": "ftp://ftp.nist.gov/",
"tag": "invalid"
}]
try:
keychain = read_keychain(keychain_file, create_default=False)
entries = get_auth_entries("https://raw.githubusercontent.com/",
keychain)
self.assertTrue(entries)
updated_keychain = update_keychain(deleted_entries,
keychain_file=keychain_file,
delete=True)
logger.info("Updated keychain: %s" % json.dumps(updated_keychain))
self.assertTrue(len(updated_keychain) == 3)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_update_keychain_del_multi(self):
logger.info(self.getTestHeader('test update keychain del multi'))
keychain_file = ospj(self.test_config_dir, 'test-keychain-8.json')
deleted_entries = [{
"uri": "https://raw.githubusercontent.com/",
"auth_type": "http-basic"
}, {
"uri": "https://raw.githubusercontent.com/",
"auth_type": "bearer-token"
}]
try:
keychain = read_keychain(keychain_file, create_default=False)
entries = get_auth_entries("https://raw.githubusercontent.com/",
keychain)
self.assertTrue(entries)
updated_keychain = update_keychain(deleted_entries,
keychain_file=keychain_file,
delete=True)
logger.info("Updated keychain: %s" % json.dumps(updated_keychain))
entries = get_auth_entries("https://raw.githubusercontent.com/",
updated_keychain)
self.assertFalse(entries)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def test_update_keychain_add_single(self):
logger.info(self.getTestHeader('test update keychain add single'))
keychain_file = ospj(self.test_config_dir, 'test-keychain-8.json')
added_entry = {
"uri": "https://foo.bar.com/",
"auth_type": "http-basic",
"auth_params": {
"auth_method": "get",
"username": "******",
"password": "******"
}
}
try:
keychain = read_keychain(keychain_file, create_default=False)
entries = get_auth_entries("https://foo.bar.com/", keychain)
self.assertFalse(entries)
updated_keychain = update_keychain(added_entry,
keychain_file=keychain_file)
logger.info("Updated keychain: %s" % json.dumps(updated_keychain))
entries = get_auth_entries("https://foo.bar.com/",
updated_keychain)
self.assertTrue(len(entries) == 1)
except Exception as e:
self.fail(bdbag.get_typed_exception(e))
def generate_remote_file_manifest(args):
keychain_file = args.keychain_file if args.keychain_file else DEFAULT_KEYCHAIN_FILE
auth = read_keychain(keychain_file)
with open(args.output_file, 'w') as rfm_file, open(args.input_file,
'r') as input_file:
rfm = list()
for url in input_file.readlines():
rfm_entry = dict()
logger.debug("Processing input URL %s" % url)
try:
headers = headForHeaders(url, auth, raise_for_status=True)
except Exception as e:
logging.warning("HEAD request failed for URL [%s]: %s" %
(url, gte(e)))
continue
length = headers.get("Content-Length")
content_type = headers.get("Content-Type")
content_disposition = headers.get("Content-Disposition")
md5 = headers.get("Content-MD5")
if md5:
md5 = decodeBase64toHex(md5)
sha256 = headers.get("Content-SHA256")
if sha256:
sha256 = decodeBase64toHex(sha256)
# if content length or both hash values are missing, there is a problem
if not length:
logging.warning("Could not determine Content-Length for %s" %
url)
if not (md5 or sha256):
logging.warning(
"Could not locate an MD5 or SHA256 hash for %s" % url)
# try to construct filename using content_disposition, if available, else fallback to the URL path fragment
filepath = urlsplit(url).path
filename = os.path.basename(filepath).split(":")[0] if not content_disposition else \
parse_content_disposition(content_disposition)
subdir = args.base_payload_path if args.base_payload_path else ""
output_path = ''.join(
[subdir, os.path.dirname(filepath), "/", filename])
rfm_entry['url'] = url
rfm_entry['length'] = length
rfm_entry['filename'] = output_path
if md5:
rfm_entry['md5'] = md5
if sha256:
rfm_entry['sha256'] = sha256
if content_type:
rfm_entry["content_type"] = content_type
rfm_entry.update({
"metadata": {
"title": os.path.basename(rfm_entry["filename"])
}
})
if args.streaming_json:
rfm_file.writelines(''.join([json.dumps(rfm_entry), '\n']))
else:
rfm.append(rfm_entry)
if not args.streaming_json:
rfm_file.write(json.dumps(rfm, indent=4))
logger.info("Successfully generated remote file manifest: %s" %
args.output_file)
def create_rfm_from_url_list(args):
keychain_file = args.keychain_file if args.keychain_file else DEFAULT_KEYCHAIN_FILE
auth = read_keychain(keychain_file)
with open(args.output_file, 'w') as rfm_file, open(args.input_file,
'r') as input_file:
rfm = list()
for url in input_file.readlines():
rfm_entry = dict()
url = url.strip()
logger.debug("Processing input URL %s" % url)
try:
headers = head_for_headers(url, auth, raise_for_status=True)
except Exception as e:
logging.warning("HEAD request failed for URL [%s]: %s" %
(url, gte(e)))
continue
logger.debug("Result headers: %s" % headers)
length = headers.get("Content-Length")
content_type = headers.get("Content-Type")
content_disposition = headers.get("Content-Disposition")
md5_header = args.md5_header if args.md5_header else "Content-MD5"
md5 = headers.get(md5_header)
md5 = get_checksum_from_string_list("md5", md5)
if md5 and not args.disable_hash_decode_base64:
rfm_entry["md5_base64"] = md5
md5 = decode_base64_to_hex(md5)
rfm_entry["md5"] = md5
sha256_header = args.sha256_header if args.sha256_header else "Content-SHA256"
sha256 = headers.get(sha256_header)
sha256 = get_checksum_from_string_list("sha256", sha256)
if sha256 and not args.disable_hash_decode_base64:
rfm_entry["sha256_base64"] = sha256
sha256 = decode_base64_to_hex(sha256)
rfm_entry["sha256"] = sha256
# if content length or both hash values are missing, there is a problem
if not length:
logging.warning("Could not determine Content-Length for %s" %
url)
if not (md5 or sha256):
logging.warning(
"Could not locate an MD5 or SHA256 hash for %s" % url)
# try to construct filename using content_disposition, if available, else fallback to the URL path fragment
filepath = urlsplit(url).path
filename = os.path.basename(filepath).split(":")[0] if not content_disposition else \
parse_content_disposition(content_disposition)
subdir = args.base_payload_path if args.base_payload_path else ""
output_path = ''.join(
[subdir, os.path.dirname(filepath), "/", filename])
rfm_entry['url'] = url
rfm_entry['length'] = length
rfm_entry['filename'] = output_path.lstrip("/")
if content_type:
rfm_entry["content_type"] = content_type
if not filter_dict(args.filter, rfm_entry):
continue
if args.streaming_json:
rfm_file.writelines(''.join(
[json.dumps(rfm_entry, sort_keys=True), '\n']))
else:
rfm.append(rfm_entry)
if not args.streaming_json:
rfm_file.write(
json.dumps(deduplicate_rfm_entries(rfm),
sort_keys=True,
indent=2))
logger.info("Successfully created remote file manifest: %s" %
args.output_file)