def test_fully_contained(self):
intervals = Intervals()
intervals.add(Interval(0, 4))
intervals.add(Interval(3, 6))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(1, 2)))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(1, 4)))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(0, 4)))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(0, 3)))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(3, 4)))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(3, 6)))
self.assertTrue(
intervals.fully_contained_in_subinterval(Interval(4, 5)))
self.assertFalse(
intervals.fully_contained_in_subinterval(Interval(0, 10)))
self.assertFalse(
intervals.fully_contained_in_subinterval(Interval(0, 5)))
self.assertFalse(
intervals.fully_contained_in_subinterval(Interval(-1, 4)))
self.assertFalse(
intervals.fully_contained_in_subinterval(Interval(3, 1000)))
self.assertFalse(
intervals.fully_contained_in_subinterval(Interval(-100, 1000)))
def test_intervals_interacting(self):
intervals = Intervals()
intervals.add(Interval(0, 4))
intervals.add(Interval(3, 6))
self.assertEqual(list(intervals.find_interacting(Interval(-10, -5))),
[])
self.assertEqual(list(intervals.find_interacting(Interval(-10, 0))),
[])
self.assertEqual(list(intervals.find_interacting(Interval(-10, 1))),
[intervals[0]])
self.assertEqual(list(intervals.find_interacting(Interval(-10, 3))),
[intervals[0]])
self.assertEqual(list(intervals.find_interacting(Interval(-10, 4))),
[intervals[0], intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(-10, 5))),
[intervals[0], intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(-10, 10))),
[intervals[0], intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(0, 10))),
[intervals[0], intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(1, 10))),
[intervals[0], intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(3, 10))),
[intervals[0], intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(4, 10))),
[intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(5, 10))),
[intervals[1]])
self.assertEqual(list(intervals.find_interacting(Interval(6, 10))), [])
self.assertEqual(list(intervals.find_interacting(Interval(2, 5))),
[intervals[0], intervals[1]])
def test_no_overlapping_constraint(self):
intervals = Intervals(allow_overlapping=False)
intervals.add(Interval(0, 4))
with self.assertRaises(IntervalConstraintException):
intervals.add(Interval(0, 4))
with self.assertRaises(IntervalConstraintException):
intervals.add(Interval(-10, 10))
def test_interval_fully_contained_subinterval(self):
intervals = Intervals()
intervals.add(Interval(100, 1285))
new = Interval(287, 407)
self.assertTrue(intervals.fully_contained_in_subinterval(new))
def test_no_identical_constraint(self):
intervals = Intervals(allow_identical=False)
intervals.add(Interval(0, 4))
with self.assertRaises(IntervalConstraintException):
intervals.add(Interval(0, 4))
intervals.add(Interval(-10, 10))
def test_no_constraints(self):
intervals = Intervals()
intervals.add(Interval(0, 4))
intervals.add(Interval(0, 4))
intervals.add(Interval(-10, 10))
def test_intervals_enumerate(self):
intervals = Intervals()
intervals.add(Interval(3, 5))
intervals.add(Interval(0, 4))
self.assertEqual(list(intervals.enumerate_members()),
[0, 1, 2, 3, 3, 4])
class ActionScrape(BaseAction):
_DERHandler = collections.namedtuple("DERHandler", [ "asn1_spec", "data_type", "extension", "pem_marker", "sanity_check_fn", "precedence" ])
_PEM_BEGIN = re.compile("^-----BEGIN (?P<marker>[ A-Za-z0-9]+)-----")
_MARKERS = {
"CERTIFICATE": "crt",
"OPENSSH PRIVATE KEY": "openssh_key",
"DSA PRIVATE KEY": "dsa_key",
"RSA PRIVATE KEY": "rsa_key",
"EC PRIVATE KEY": "ec_key",
"PUBLIC KEY": "pubkey",
"X509 CRL": "crl",
"CERTIFICATE REQUEST": "csr",
"NEW CERTIFICATE REQUEST": "csr",
}
_DER_CLASSES = {
handler_class.data_type: handler_class for handler_class in (
_DERHandler(asn1_spec = rfc2459.Certificate(), data_type = "crt", extension = "der", pem_marker = "CERTIFICATE", sanity_check_fn = None, precedence = 10),
_DERHandler(asn1_spec = rfc2437.RSAPrivateKey(), data_type = "rsa_key", extension = "der", pem_marker = "RSA PRIVATE KEY", sanity_check_fn = _DERSanityCheck.check_rsa_key, precedence = 20),
_DERHandler(asn1_spec = rfc2459.DSAPrivateKey(), data_type = "dsa_key", extension = "der", pem_marker = "DSA PRIVATE KEY", sanity_check_fn = None, precedence = 20),
_DERHandler(asn1_spec = rfc2459.SubjectPublicKeyInfo(), data_type = "pubkey", extension = "der", pem_marker = "PUBLIC KEY", sanity_check_fn = None, precedence = 30),
_DERHandler(asn1_spec = x509sak.ASN1Models.ECPrivateKey(), data_type = "ec_key", extension = "der", pem_marker = "EC PRIVATE KEY", sanity_check_fn = _DERSanityCheck.check_ec_key, precedence = 20),
_DERHandler(asn1_spec = x509sak.ASN1Models.PFX(), data_type = "pkcs12", extension = "p12", pem_marker = None, sanity_check_fn = None, precedence = 0),
_DERHandler(asn1_spec = x509sak.ASN1Models.DSASignature(), data_type = "dsa_sig", extension = "der", pem_marker = None, sanity_check_fn = _DERSanityCheck.check_dsa_sig, precedence = 40),
)}
handler_classes = sorted(list(_DER_CLASSES.keys()))
def __init__(self, cmdname, args):
BaseAction.__init__(self, cmdname, args)
# Plausibilize input parameters
kwargs_checker = KwargsChecker(optional_arguments = set(self._DER_CLASSES.keys()))
kwargs_checker.check(self._args.include_dertype, hint = "DER classes to be included")
kwargs_checker.check(self._args.exclude_dertype, hint = "DER classes to be excluded")
# Plausibilize output directory
if os.path.exists(self._args.outdir) and (not self._args.force):
raise Exception("Directory %s already exists. Remove it first or use --force." % (self._args.outdir))
try:
os.makedirs(self._args.outdir)
except FileExistsError:
pass
# Determine active DERHandler classes
if len(self._args.include_dertype) == 0:
active_der_types = set(self._DER_CLASSES.keys())
else:
active_der_types = set(self._args.include_dertype)
active_der_types -= set(self._args.exclude_dertype)
self._active_der_types = [ self._DER_CLASSES[class_name] for class_name in active_der_types ]
self._active_der_types.sort(key = lambda handler: (handler.precedence, handler.data_type))
self._stats = ActionScrapeStats(self._args)
self._stats.set_active_der_types([ handler_class.data_type for handler_class in self._active_der_types ])
self._matches = Intervals()
self._hashes = set()
engine = ScrapeEngine(self._args.filename)
if not self._args.no_pem:
engine.search(self._find_pem, b"-----BEGIN ", min_length = 52, max_length = 32 * 1024)
if (not self._args.no_der) and (len(self._active_der_types) > 0):
self._log.debug("Looking for %d DER type(s): %s", len(self._active_der_types), ", ".join(handler.data_type for handler in self._active_der_types))
engine.search(self._find_der, bytes.fromhex("30"), min_length = 2, max_length = 32 * 1024)
end_offset = engine.commence(start_offset = self._args.seek_offset, length = self._args.analysis_length, progress_callback = self._progress_callback)
self._stats.finish(end_offset)
self._stats.dump()
if self._args.write_json is not None:
JSONTools.write_to_file(self._stats.as_dict(), self._args.write_json)
def _progress_callback(self, position, total_length, elapsed_secs):
self._log.debug("Scan at %.0f MiB of %.0f MiB, %.1f%%. Average speed %.1f MiB/sec", position / 1024 / 1024, total_length / 1024 / 1024, position / total_length * 100, position / 1024 / 1024 / elapsed_secs)
def _is_nested_match(self, offset, length):
if self._args.extract_nested:
# Completely disregard if we've already captured this.
return False
interval = Interval.begin_length(offset, length)
if self._matches.fully_contained_in_subinterval(interval):
# We already have this match.
return True
else:
self._matches.add(interval)
return False
def _is_known_blob(self, data):
if self._args.allow_non_unique_blobs:
# We record the exact same file twice, always.
return False
blob_hash = hashlib.sha256(data).digest()
if blob_hash in self._hashes:
return True
else:
self._hashes.add(blob_hash)
return False
def _record_finding(self, offset, data_type, extension, data, encode_pem_marker = None, orig_extension = None):
if orig_extension is None:
orig_extension = data_type
if len(data) == 0:
self._stats.record_finding(offset, len(data), data_type, orig_extension, "discard:zero_length")
return
if self._is_nested_match(offset, len(data)):
self._stats.record_finding(offset, len(data), data_type, orig_extension, "discard:nested")
self._log.debug("Found %s/%s at offset 0x%x, length %d bytes, not recording nested match.", data_type, orig_extension, offset, len(data))
return
if self._is_known_blob(data):
self._stats.record_finding(offset, len(data), data_type, orig_extension, "discard:non-unique")
self._log.debug("Found %s/%s at offset 0x%x, length %d bytes, not recording non-unique match.", data_type, orig_extension, offset, len(data))
return
filename_args = {
"otype": orig_extension,
"type": data_type,
"offset": offset,
"ext": extension,
}
filename = self._args.outdir + "/" + (self._args.outmask % filename_args)
self._stats.record_finding(offset, len(data), data_type, orig_extension, "written", filename)
self._log.info("Found %s/%s at offset 0x%x, length %d bytes, saved as %s", data_type, orig_extension, offset, len(data), filename)
if encode_pem_marker is not None:
output_data = (PEMDataTools.data2pem(data, encode_pem_marker) + "\n").encode()
else:
output_data = data
with open(filename, "wb") as f:
f.write(output_data)
def _find_pem(self, offset, data):
self._stats.pem_potential_match()
textdata = data.decode("ascii", errors = "ignore")
result = self._PEM_BEGIN.match(textdata)
if result is None:
return
result = result.groupdict()
marker = result["marker"]
full_re = re.compile("-----BEGIN %s-----(?P<pem_data>.*?)-----END %s-----" % (marker, marker), flags = re.DOTALL | re.MULTILINE)
result = full_re.match(textdata)
if result is None:
return
result = result.groupdict()
pem_data = result["pem_data"]
pem_data = pem_data.replace("\r", "")
pem_data = pem_data.replace("\n", "")
pem_data = pem_data.replace("\t", "")
pem_data = pem_data.replace(" ", "")
der_data = base64.b64decode(pem_data)
self._stats.pem_successful_decode()
data_type = self._MARKERS.get(marker, "unknown")
self._record_finding(offset = offset, data_type = data_type, extension = "pem", data = der_data, encode_pem_marker = marker)
def _find_der(self, offset, data):
self._stats.der_potential_match()
for der_candidate in self._active_der_types:
try:
self._stats.der_attempt_decode()
(asn1, tail) = pyasn1.codec.der.decoder.decode(data, asn1Spec = der_candidate.asn1_spec)
if len(tail) == 0:
asn1_data = data
else:
asn1_data = data[:-len(tail)]
self._stats.der_successful_decode()
if (not self._args.disable_der_sanity_checks) and (der_candidate.sanity_check_fn is not None):
# We want sanity checks enabled and for the successfully
# deserialized ASN.1 blob there is a handler registered.
# Execute it (it'll throw an exception on failure, which
# we'll catch).
der_candidate.sanity_check_fn(asn1)
self._stats.der_passed_plausibility()
if self._args.keep_original_der or (der_candidate.pem_marker is None):
# Should not or cannot re-encode as PEM, write DER file
self._record_finding(offset = offset, data_type = der_candidate.data_type, extension = der_candidate.extension, data = asn1_data)
else:
self._record_finding(offset = offset, data_type = der_candidate.data_type, extension = "pem", data = asn1_data, encode_pem_marker = der_candidate.pem_marker, orig_extension = der_candidate.extension)
except pyasn1.error.PyAsn1Error as e:
pass
except _DERSanityCheck.SanityCheckFailedException as e:
self._log.debug("Potential %s blob encountered at offset 0x%x, but failed sanity check: %s", der_candidate.data_type, offset, str(e))
self._stats.der_failed_plausibility()