def create_self_signed_cert():
"""Create self signed key+cert.pem."""
print "Generating certificate with OpenSSL..."
# Folder to hold all client certificates
if not os.path.exists(KEYS_FOLDER): os.makedirs(KEYS_FOLDER)
# OpenSSL will ask you some arbitrary set of questions, all of which are irrelevat for self-signed certificates.
# This feeds a large set of newlines in an attempt to brute-force ignore its prompts.
input = []
for x in range(0, 50):
input.append("\n")
# Generate individual files
sh.openssl("req", "-x509", "-newkey", "rsa:2048", "-keyout", KEY_FILE, "-out", CERT_FILE, "-days", "999", "-nodes", _in=input)
if not (os.path.isfile(KEY_FILE) and os.path.isfile(CERT_FILE)):
print "OpenSSL failed to generate both a " + KEY_FILE + " and a " + CERT_FILE + "."
sys.exit(1)
# Combine for our purposes
key = open(KEY_FILE).read()
cert = open(CERT_FILE).read()
combined = open(KEY_CERT_COMBINED_FILE, "w")
combined.write(key + cert)
combined.close()
if os.path.exists(KEY_CERT_COMBINED_FILE):
print 'generated %s, %s and %s' % (KEY_FILE, CERT_FILE, KEY_CERT_COMBINED_FILE)
def __init__(self, encrypted_key, keypair):
"""Creates an encryption key, using the given keypair to encrypt/decrypt it.
The plaintext version of this key is kept in a temporary file that will be securely
destroyed upon this object becoming garbage collected.
:param encrypted_key: the encrypted version of this key is kept in this file: if it
does not exist, it will be created when this key is saved
:type encrypted_key: str
:param keypair: a tuple containing the (private, public) key pair that will be used to
decrypt and encrypt (respectively) this key.
:type keypair: collections.namedtuple (Keypair)
"""
self._plaintext = mkstemp()[1]
self.encrypted = encrypted_key
self.key_pair = keypair
if not os.path.exists(encrypted_key):
openssl('rand', '32', '-out', self._plaintext)
else:
with open(encrypted_key, 'rb') as secret:
openssl('rsautl',
'-decrypt',
'-inkey',
keypair.private,
_in=secret,
_out=self._plaintext)
def _save(self):
""" Encrypts the contents of the key and writes it out to disk. """
if not os.path.exists(self.key_pair.public):
raise RuntimeError("Encryption key file '%s' not found" %
self.key_pair.public)
with open(self._plaintext, 'rb') as selfkey:
openssl('rsautl',
'-encrypt',
'-pubin',
'-inkey',
self.key_pair.public,
_in=selfkey,
_out=self.encrypted)
def get_fingerprint(cert):
'''
Get the MD5 fingerprint of an SSL certificate.
>>> web_cert = '/var/local/projects/goodcrypto/server/data/web/security/web.ca.crt'
>>> with open(web_cert) as f:
... fingerprint = get_fingerprint(f.read())
... fingerprint is not None
True
'''
fingerprint = None
try:
result = sh.openssl('x509', '-noout', '-fingerprint', _in=cert)
if IS_PY2:
result_stdout = result.stdout
else:
result_stdout = result.stdout.decode()
m = re.match('SHA1 Fingerprint=(.*)', result_stdout)
if m:
fingerprint = m.group(1).strip()
else:
log('fingerprint result stdout: {}'.format(result_stdout))
log('fingerprint result stderr: {}'.format(result.stderr))
except:
log(format_exc())
return fingerprint
def get_valid_dates(cert):
'''
Get the dates an SSL certificate is valid.
>>> web_cert = '/var/local/projects/goodcrypto/server/data/web/security/web.ca.crt'
>>> with open(web_cert) as f:
... not_before, not_after = get_valid_dates(f.read())
... not_before is not None
... not_after is not None
True
True
'''
not_before = not_after = None
try:
result = sh.openssl('x509', '-noout', '-dates', _in=cert)
if IS_PY2:
result_stdout = result.stdout
else:
result_stdout = result.stdout.decode()
m = re.match('notBefore=(.*?)\nnotAfter=(.*)', result_stdout)
if m:
not_before = m.group(1).strip()
not_after = m.group(2).strip()
else:
log('dates result stdout: {}'.format(result_stdout))
log('dates result stderr: {}'.format(result.stderr))
except:
log(format_exc())
return not_before, not_after
def get_issued_to(cert):
'''
Get to whom an SSL certificate was issued.
>>> web_cert = '/var/local/projects/goodcrypto/server/data/web/security/web.ca.crt'
>>> with open(web_cert) as f:
... get_issued_to(f.read())
'/O=GoodCrypto Private Server Certificate Authority/CN=goodcrypto.private.server.proxy'
'''
issued_to = None
try:
result = sh.openssl('x509', '-noout', '-subject', _in=cert)
if IS_PY2:
result_stdout = result.stdout
else:
result_stdout = result.stdout.decode()
m = re.match('subject=(.*)', result_stdout)
if m:
issued_to = m.group(1).strip()
else:
log('issued_to result stdout: {}'.format(result_stdout))
log('issued_to result stderr: {}'.format(result.stderr))
except:
log(format_exc())
return issued_to
def create_self_certificate_authority(force=False):
"""Create our own certificate authority."""
if os.path.exists(ROOT_CERT_COMBINED_FILE) and not force:
print('\n\nWARNING! WARNING!')
print('\n%s already exists. Replacing this file will invalidate')
print('any existing client certificates. You will need to recreate')
print('all client certificates.')
confirm = raw_input('Are you sure you wish to continue? [y/N]: ') or 'n'
if confirm.lower() == 'y':
create_self_certificate_authority(force=True)
input_ = ['\n'] * 50
sh.openssl('genrsa', '-out', ROOT_KEY_FILE, '2048')
sh.openssl('req', '-x509', '-new', '-nodes', '-key', ROOT_KEY_FILE, '-days', '999', '-out', ROOT_CERT_FILE, _in=input_)
# now join two to give to nginx
key = open(ROOT_KEY_FILE).read()
cert = open(ROOT_CERT_FILE).read()
combined = open(ROOT_CERT_COMBINED_FILE, "w")
combined.write(key + cert)
combined.close()
def _decrypt(self):
""" Performs the decryption of an encrypted file.
This is the reverse operation of ```encrypt()``` executing virtually an identical
```openssl``` command, with the in/out roles reversed and adding a ```-d``` flag.
:return `True` if the deryption was successful
:rtype bool
"""
self._outfile = os.path.join(self.dest, self.plain_file)
self._infile = self.encrypted_file
self._log.info("Decrypting file '%s' to '%s'", self.encrypted_file, self._outfile)
with open(self.encrypted_file, 'rb') as enc_file:
openssl('enc', '-aes-256-cbc', '-d', '-pass',
'file:{secret}'.format(secret=self.secret.keyfile),
_in=enc_file,
_out=self._outfile)
self._log.info("File '%s' decrypted to '%s'", self.encrypted_file, self._outfile)
return True
def gen_cert(domain, dirname, private_key_name, public_cert_name, days):
''' Generate the public certificate. '''
log('generating certificate')
private_key = os.path.join(dirname, 'private', private_key_name)
public_cert = os.path.join(dirname, public_cert_name)
csr = os.path.join(dirname, '{}.csr'.format(domain))
sh.openssl('x509', '-req', '-days', days, '-in', csr, '-signkey', private_key, '-out', public_cert)
assert os.path.exists(public_cert), 'could not generate {}'.format(public_cert)
os.remove(csr)
# only the owner should be able to read the private key
sh.chmod('u+r', private_key)
sh.chmod('u-wx', private_key)
sh.chmod('go-rwx', private_key)
# everyone can read the public certificate
sh.chmod('ugo+r', public_cert)
sh.chmod('ugo-wx', public_cert)
def _encrypt(self):
"""Performs the encryption step.
This uses a combination of the `sh` module and OpenSSL to execute the following command
line::
openssl enc -aes-256-cbc -pass file:(secret) < plain_file > dest/plain_file.enc
:return `True` if the encryption was successful
:rtype bool
"""
self._outfile = os.path.join(self.dest, self.encrypted_file)
self._infile = self.plain_file
self._log.info("Encrypting '%s' to '%s'", self.plain_file, self._outfile)
with open(self.plain_file, 'rb') as plain_file:
openssl('enc', '-aes-256-cbc', '-pass',
'file:{secret}'.format(secret=self.secret.keyfile),
_in=plain_file,
_out=self._outfile)
self._log.info("File '%s' encrypted to '%s'", self.plain_file, self._outfile)
return True
def create_client_cert(drone_name):
"""
Create a new client certificate from the specified drone.
Each of the signed certs must have a complete DN to be valid, including common name.
However, the common name does not need to match the match with the drone. This is a
reminder that nginx is verifying the certificate is signed by a trusted CA, it does not
performs reverse look-up to verify the hostname.
"""
input_ = ['\n'] * 5 + ['localhost\n'] + (['\n'] * 30)
drone_key = os.path.join('persistent', 'keys', 'client-%s-key.pem' % drone_name)
drone_cert = os.path.join('persistent', 'keys', 'client-%s-cert.pem' % drone_name)
drone_csr = os.path.join('persistent', 'keys', 'client-%s.csr' % drone_name)
drone_combined = os.path.join('persistent', 'keys', 'client-%s-key+cert.pem' % drone_name)
sh.openssl('genrsa', '-out', drone_key, '2048')
sh.openssl('req', '-new', '-key', drone_key, '-out', drone_csr, _in=input_)
if not os.path.exists(ROOT_SRL_FILE):
print 'creating new CA serial file'
cmd = ['x509', '-req', '-in', drone_csr, '-CA', ROOT_CERT_FILE, '-CAkey', ROOT_KEY_FILE, '-CAcreateserial', '-out', drone_cert, '-days', '999']
else:
print 'reusing exisitng CA serial file'
cmd = ['x509', '-req', '-in', drone_csr, '-CA', ROOT_CERT_FILE, '-CAkey', ROOT_KEY_FILE, '-CAserial', ROOT_SRL_FILE, '-out', drone_cert, '-days', '999']
sh.openssl(cmd)
key = open(drone_key).read()
cert = open(drone_cert).read()
combined = open(drone_combined, "w")
combined.write(key + cert)
combined.close()
print '\ngenerated %s' % drone_combined
# After signing, the CSR is useless
os.remove(drone_csr)
def get_certs(subj: Optional[str] = None, filename: str = "cert"):
"""Creates a certificate for mitmproxy using openssl. If exists
returns the previously created one
Keyword Arguments:
subj {[type]} -- [description] (default: {None})
dest {[type]} -- [description] (default: {None})
filename {str} -- [description] (default: {'cert'})
Returns:
[type] -- [description]
"""
if subj is None:
subj = (
"/C=GB/ST=London/L=London/O=Global Security/OU=IT Department/CN=example.com"
)
base = CONFIG_DIR
key = base / f"{filename}.key"
if not key.exists():
logger.info(f"Generating {key}")
sh.openssl(shlex.split(f"genrsa -out {key.name} 2048"), _cwd=CONFIG_DIR)
crt = base / f"{filename}.crt"
if not crt.exists():
logger.info(f"Generating {crt}")
sh.openssl(
shlex.split(
f"req -new -x509 -key {key.name} -out {crt.name} -subj '{subj}'"
),
_cwd=str(base),
)
pem = base / f"{filename}.pem"
if not pem.exists():
logger.info(f"Generating {pem}")
with pem.open("w") as fp:
for pth in (key, crt):
fp.write(pth.read_text())
return pem
def get_fingerprint():
'''Get the fingerprint of the web certificate.'''
try:
# get the fingerprint
result = sh.openssl('x509', '-fingerprint', '-in', CA_FILE, '-md5')
log('result: {}'.format(result.exit_code))
if result.exit_code == 0:
lines = str(result.stdout).split('\n')
fingerprint = lines[0].replace('SHA1 Fingerprint=', '')
else:
fingerprint = result.stderr
except Exception:
log(format_exc())
fingerprint = 'Unable to get fingerprint due to an unexpected error'
return fingerprint
def get_hash(cert):
'''
Get the hash of an SSL certificate.
>>> web_cert = '/var/local/projects/goodcrypto/server/data/web/security/web.ca.crt'
>>> with open(web_cert) as f:
... cert_hash = get_hash(f.read())
... cert_hash is not None
True
'''
cert_hash = None
try:
result = sh.openssl('x509', '-noout', '-hash', _in=cert)
if IS_PY2:
result_stdout = result.stdout
else:
result_stdout = result.stdout.decode()
cert_hash = result_stdout.strip()
except:
log(format_exc())
return cert_hash
def verify_certificate(hostname, port, ca_certs_dir=None):
'''
Verify a certificate is valid. Compare against openssl's published certs.
Debian Jessie openssl does not support proxies, so we can't specify a tor proxy directly.
This function uses sh, so syr.net.torify() doesn't work.
>>> ok, _, _ = verify_certificate('google.com', 443)
>>> ok
True
>>> ok, _, error_message = verify_certificate('goodcrypto.private.server', 443)
>>> ok
False
>>> if IS_PY2:
... error_message == u'self signed certificate'
... else:
... error_message == 'self signed certificate'
True
>>> ok, _, error_message = verify_certificate('www.mjvmobile.com.br', 443)
>>> ok
False
>>> if IS_PY2:
... error_message == u'certificate has expired'
... else:
... error_message == 'certificate has expired'
True
'''
def extract_cert(response):
cert = None
i = response.find('-----BEGIN CERTIFICATE-----')
if i > 0:
temp_cert = response[i:]
i = temp_cert.find('-----END CERTIFICATE-----')
if i > 0:
cert = temp_cert[:i+len('-----END CERTIFICATE-----')]
return cert
def verify_date(cert):
ok = True
error_message = None
not_before, not_after = get_dates(cert)
try:
after_date = datetime.strptime(not_after, '%b %d %H:%M:%S %Y %Z')
except:
try:
after_date = datetime.strptime(not_after, '%b %d %H:%M:%S %Y %z')
except:
after_date = datetime.now() + timedelta(days=1)
if after_date < datetime.now():
ok = False
error_message = '{} {}'.format(EXPIRED_CERT_ERR_MSG, not_after)
return ok, error_message
ok = True
error_message = ''
cert = None
server = '{}:{}'.format(hostname, port)
log('verify cert for {}'.format(server))
if ca_certs_dir is None:
ca_certs_dir = get_ca_certs_dir()
log('ca certs dir: {}'.format(ca_certs_dir))
try:
# s_client waits for stdin after connecting, so we provide a short stdin
# _in=' ' instead of _in='' because apparently sh does something like
# checks '_in' instead of '_in is None'
result = sh.openssl('s_client', '-connect', server, '-CApath', ca_certs_dir, _in=' ')
except sh.ErrorReturnCode as erc:
ok = False
try:
stderr = erc.stderr.strip()
log('verify failed stderr:\n{}'.format(stderr))
# parse 'connect: No route to host\nconnect:errno=22'
# to 'connect: No route to host'
error_message = stderr.split('\n')[0].strip()
except:
error_message = erc # 'Unable to verify SSL certificate'
log(erc)
else:
if IS_PY2:
result_stderr = result.stderr
else:
result_stderr = result.stderr.decode()
log('verify result stderr:\n{}'.format(result_stderr))
lines = result_stderr.split('\n')
return_code = None
for line in lines:
if line.startswith('verify return:'):
return_code = line[len('verify return:'):]
elif line.startswith('verify error:'):
m = re.match('verify error:num=(\d+):(.*)', line)
if m:
return_code = m.group(1)
error_message += m.group(2)
else:
return_code = -1
error_message = line
# get the certificate so we can do additional verification
if IS_PY2:
cert = extract_cert(result.stdout)
else:
cert = extract_cert(result.stdout.decode())
# it seems like we're never able to verify the local issuer so ignore the error
if return_code == '0' and error_message == 'unable to get local issuer certificate':
error_message = None
if error_message is not None and len(error_message) > 0:
ok = False
log('error verifying {} certificate: {}'.format(hostname, error_message))
else:
error_message = None
if return_code == '0' and error_message is None:
ok, error_message = verify_date(cert)
log('cert ok: {}'.format(ok))
return ok, cert, error_message
for segment in segments:
if segment['key'] is None:
continue
segment['encrypted'] = segment['file']
segment['decrypted'] = '%s.decrypted' % segment['file']
if os.path.exists(segment['decrypted']):
segment['file'] = segment['decrypted']
continue
else:
print "decrypting %s" % segment['file']
key = binascii.hexlify(open(segment['key']['file']).read())
#openssl aes-128-cbc -d -K $(hexdump -e '/1 "%02X"' key) -iv 0 -nosalt -in $i -out 12215-dec/$b
openssl("aes-128-cbc", "-d", "-K", key, "-iv", 0, "-nosalt", "-in", segment['encrypted'], "-out", segment['decrypted'])
segment['file'] = segment['decrypted']
tsfile = "%s/combined.ts" % options.scratch
if not os.path.isfile(tsfile):
tshandle = open(tsfile, "w", -1)
tshandle.seek(0)
for segment in segments:
print segment['file']
segmenthandle = open(segment['file'], "r", -1)
while 1:
buffer = segmenthandle.read()
if len(buffer) == 0:
break
tshandle.write(buffer)
segmenthandle.close()
