def download_and_extract_openssl():
"""Download openssl distribution and extract it to OPENSSL_BUILD_DIR."""
if os.path.isdir(OPENSSL_BUILD_DIR):
shutil.rmtree(OPENSSL_BUILD_DIR, ignore_errors=True)
mkpath(OPENSSL_BUILD_DIR)
# Download openssl
log.info("Downloading OpenSSL from: {}".format(CONFIG['openssl_dist']))
temp_filename = os.path.join(tempfile.mkdtemp(), 'tempdata')
cmd = [
'/usr/bin/curl', '--show-error', '--no-buffer', '--fail',
'--progress-bar', '--speed-time', '30', '--location', '--url',
CONFIG['openssl_dist'], '--output', temp_filename
]
# We are calling os.system so we can get download progress live
rc = runner.system(cmd)
if rc == 0 or rc is True:
log.debug("OpenSSL download successfully")
else:
log.error("OpenSSL download failed with exit code: '{}'".format(rc))
sys.exit(1)
# Verify openssl download hash
download_hash = hash_helper.getsha256hash(temp_filename)
config_hash = CONFIG['openssl_dist_hash']
if download_hash != config_hash:
log.error("Hash verification of OpenSSL download has failed. Download "
"hash of '{}' does not match config hash '{}'".format(
download_hash, config_hash))
sys.exit(1)
else:
log.detail("Hash verification of OpenSSL successfully")
# Extract openssl to the openssl_build_dir
log.info("Extracting OpenSSL...")
cmd = [
'/usr/bin/tar', '-xf', temp_filename, '-C', OPENSSL_BUILD_DIR,
'--strip-components', '1'
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Extraction completed successfullyly")
else:
log.error("Extraction has failed: {}".format(out[1]))
os.remove(temp_filename)
def dl_and_extract_python(dist_url, dist_hash):
"""Download Python distribution and extract it to PYTHON_BUILD_DIR."""
if os.path.isdir(PYTHON_BUILD_DIR):
shutil.rmtree(PYTHON_BUILD_DIR, ignore_errors=True)
mkpath(PYTHON_BUILD_DIR)
# Download Python
log.info("Downloading Python from: {}".format(dist_url))
temp_filename = os.path.join(tempfile.mkdtemp(), 'tempdata')
cmd = [
'/usr/bin/curl', '--show-error', '--no-buffer', '--fail',
'--progress-bar', '--speed-time', '30', '--location', '--url',
dist_url, '--output', temp_filename
]
# We are calling os.system so we can get download progress live
rc = runner.system(cmd)
if rc == 0 or rc is True:
log.debug("Python download successful")
else:
log.error("Python download failed with exit code: '{}'".format(rc))
sys.exit(1)
# Verify Python download hash
download_hash = hash_helper.getsha256hash(temp_filename)
config_hash = dist_hash
if download_hash != config_hash:
log.error("Hash verification of Python download has failed. Download "
"hash of '{}' does not match config hash '{}'".format(
download_hash, config_hash))
sys.exit(1)
else:
log.detail("Hash verification of Python successful")
# Extract Python to the PYTHON_BUILD_DIR
log.info("Extracting Python...")
cmd = [
'/usr/bin/tar', '-xf', temp_filename, '-C', PYTHON_BUILD_DIR,
'--strip-components', '1'
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Extraction completed successfully")
else:
log.error("Extraction has failed: {}".format(out[0]))
os.remove(temp_filename)
def patch():
"""Patch source files for the build phase."""
log.info("Creating our patch files for tlsssl...")
patch_dir = os.path.join(CURRENT_DIR, '_patch')
if not os.path.exists(patch_dir):
log.debug("Creating _patch directory...")
mkpath(patch_dir)
patch_pairs = [
# ['_patch/_ssl.c', '_src/_ssl.c', ],
['_patch/make_ssl_data.py', '_src/make_ssl_data.py'],
# ['_patch/ssl.py', '_src/ssl.py'],
]
log.detail("Create our patch files if they do not exist...")
for dest, source in patch_pairs:
if not os.path.isfile(os.path.join(CURRENT_DIR, dest)):
source = os.path.join(CURRENT_DIR, source)
diff = os.path.join(CURRENT_DIR, '_diffs',
"{}.diff".format(os.path.basename(dest)))
dest = os.path.join(CURRENT_DIR, dest)
log.debug("Patching '{}'".format(dest))
# TODO: Validate the return code and exist if something didn't work
cmd = ['/usr/bin/patch', source, diff, "-o", dest]
out = runner.Popen(cmd)
runner.pprint(out)
# Copy over the socketmodule.h file as well
if not os.path.isfile(os.path.join(patch_dir, "socketmodule.h")):
log.debug("Copying 'socketmodule.h' to the _patch dir")
source = os.path.join(CURRENT_DIR, "_src", "socketmodule.h")
shutil.copy(source, os.path.realpath(os.path.join(patch_dir)))
if not os.path.isfile(os.path.join(patch_dir, "_ssl.c")):
log.debug("Copying '_ssl.c' to the _patch dir")
source = os.path.join(CURRENT_DIR, "_src", "_ssl.c")
shutil.copy(source, os.path.realpath(os.path.join(patch_dir)))
if not os.path.isfile(os.path.join(patch_dir, "ssl.py")):
log.debug("Copying 'ssl.py' to the _patch dir")
source = os.path.join(CURRENT_DIR, "_src", "ssl.py")
shutil.copy(source, os.path.realpath(os.path.join(patch_dir)))
log.detail("All patch files are created...")
def current_certs():
"""
Include current SystemRoot certs with OpenSSL.
This helps work around a limitation with bundling your own version of
OpenSSL. We copy the certs from Apple's 'SystemRootCertificates.keychain'
into OPENSSL_BUILD_DIR/cert.pem
https://goo.gl/s6vvwl
"""
log.info("Writing the 'cert.pem' file from Apple's System Root Certs...")
cmd = [
'/usr/bin/security', 'find-certificate', '-a', '-p',
'/System/Library/Keychains/SystemRootCertificates.keychain'
]
out = runner.Popen(cmd)
# Take the command output, out[0], and write the file to cert.pem
cert_path = os.path.join(PKG_PAYLOAD_DIR, BASE_INSTALL_PATH_S, 'openssl',
'cert.pem')
try:
f = open(cert_path, "w")
f.write(out[0])
f.close()
except (IOError) as e:
log.error("Unable to write 'cert.pem': {}".format(e))
def build(py_version, py_install_path, skip):
"""Build custom Python from source."""
py_major_ver = py_version.split('.')[0]
log.debug("Currently building: {}".format(py_major_ver))
# Step 1: change into our build directory
os.chdir(PYTHON_BUILD_DIR)
# Don't compile Python if the skip option is passed
if skip:
log.info("Python compile skipped due to -skip option")
return
# Step 1.5: Add extra modules
if py_major_ver == '2':
setup_dist = os.path.join(PYTHON_BUILD_DIR, 'Modules/Setup.dist')
with open(setup_dist, "a") as f:
log.debug("Adding additional modules to be included...")
f.write("_socket socketmodule.c timemodule.c\n")
f.write("_ssl _ssl.c -DUSE_SSL "
"-I{0}/include -I{0}/include/openssl -L{0}/lib "
"-lssl -lcrypto".format(OPENSSL_INSTALL_PATH))
# Step 2: Run the Configure setup of Python to set correct paths
os.chdir(PYTHON_BUILD_DIR)
if os.path.isdir(py_install_path):
shutil.rmtree(py_install_path, ignore_errors=True)
mkpath(py_install_path)
log.info("Configuring Python...")
cmd = [
'./configure',
'--prefix={}'.format(py_install_path),
# 'CPPFLAGS=-I{}/include'.format(OPENSSL_INSTALL_PATH),
# 'LDFLAGS=-L{}/lib'.format(OPENSSL_INSTALL_PATH),
'CFLAGS=-I{}/include'.format(OPENSSL_INSTALL_PATH),
'LDFLAGS=-L{}/lib'.format(OPENSSL_INSTALL_PATH),
'--enable-shared',
'--enable-toolbox-glue',
'--with-ensurepip=install',
'--enable-ipv6',
'--with-threads',
'--datarootdir={}/share'.format(py_install_path),
'--datadir={}/share'.format(py_install_path),
# '--enable-optimizations', # adding this flag will run tests
]
runner.Popen(cmd, stdout=sys.stdout)
# Step 3: compile Python. this will take a while.
# FIXME: We need to check return codes.
log.info("Compiling Python. This will take a while time...")
log.detail("Running Python make routine...")
cmd = ['/usr/bin/make']
runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
log.debug("Create some temp files thats")
log.detail("Running Python make install routine...")
cmd = ['/usr/bin/make', 'install']
runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
# Step 4: Install pip + requirements
os.chdir(os.path.join(py_install_path, 'bin'))
# Update pip to latest
log.info("Upgrading pip...")
if py_major_ver == '2':
cmd = ['./pip']
elif py_major_ver == '3':
cmd = ['./pip3']
cmd = cmd + ['install', '--upgrade', 'pip']
runner.Popen(cmd, stdout=sys.stdout)
# Install all pip modules from requirements.txt
log.info("Install requirements...")
if py_major_ver == '2':
cmd = [
'./python2.7', '-m', 'pip', 'install', '-r',
os.path.join(CURRENT_DIR, 'requirements2.txt')
]
elif py_major_ver == '3':
cmd = [
'./python3.6', '-m', 'pip', 'install', '-r',
os.path.join(CURRENT_DIR, 'requirements3.txt')
]
runner.Popen(cmd, stdout=sys.stdout)
def build():
"""Build OpenSSL from source."""
# Step 1: change into our build directory
os.chdir(OPENSSL_BUILD_DIR)
# Don't compile openssl if the skip option is passed
# Step 2: Run the Configure setup of OpenSSL to set correct paths
openssl_install = os.path.join(BASE_INSTALL_PATH, 'openssl')
log.info("Configuring OpenSSL...")
cmd = [
'./Configure',
'--prefix={}'.format(openssl_install),
'--openssldir={}'.format(openssl_install),
'darwin64-x86_64-cc',
'enable-ec_nistp_64_gcc_128',
'no-ssl2',
'no-ssl3',
'no-zlib',
'shared',
'enable-cms',
'no-comp',
]
# OpenSSL 1.0 to 1.1 has some pretty major API and build differences.
# Manage the build control with the following. NOTE: 1.1 is not
# supported at this time. Hopefully in a future release.
OLD_VERSION = None
# OpenSSL renamed this build flag so was less confusing
# https://github.com/openssl/openssl/commit/3c65577f1af1109beb8de06420efa09188981628
TMP_DIR_FLAG = None
if OPENSSL_VERSION > "1.1.0":
OLD_VERSION = False
TMP_DIR_FLAG = "DESTDIR"
else:
OLD_VERSION = True
TMP_DIR_FLAG = "INSTALL_PREFIX"
# If running 1.0 use runner.system() else runner.Popen()
if OLD_VERSION:
out = runner.system(cmd)
else:
out = runner.Popen(cmd)
log.debug("Configuring returned value: {}".format(out))
# Step 3: compile openssl. this will take a while.
# FIXME: We need to check return codes.
# This command is required for OpenSSL lower than 1.1
log.info("Compiling OpenSSL. This will take a while time...")
if OLD_VERSION:
log.detail("Running OpenSSL make depend routine...")
cmd = ['/usr/bin/make', 'depend']
proc = subprocess.Popen(cmd, bufsize=-1, stdout=sys.stdout)
(output, dummy_error) = proc.communicate()
sys.stdout.flush() # does this help?
log.detail("Running OpenSSL make routine...")
cmd = ['/usr/bin/make']
proc = subprocess.Popen(cmd, bufsize=-1, stdout=sys.stdout)
(output, dummy_error) = proc.communicate()
sys.stdout.flush() # does this help?
# log.detail("Running OpenSSL make test routine...")
# cmd = ['/usr/bin/make', 'test']
# proc = subprocess.Popen(cmd, bufsize=-1, stdout=sys.stdout)
# (output, dummy_error) = proc.communicate()
# sys.stdout.flush() # does this help?
mkpath(PKG_PAYLOAD_DIR)
log.detail("Running OpenSSL make install routine...")
cmd = [
'/usr/bin/make', '{}={}'.format(TMP_DIR_FLAG, PKG_PAYLOAD_DIR),
'install'
]
print("ran a command: {}".format(' '.join(cmd)))
out = runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
def main():
"""Build and package OpenSSL."""
parser = argparse.ArgumentParser(prog='OpenSSL setup',
description='This script will compile '
'OpenSSL 1.0.1+ and optionally create '
'a native macOS package.')
parser.add_argument('-b',
'--build',
action='store_true',
help='Compile the OpenSSL binary')
parser.add_argument('-s',
'--skip',
action='store_true',
help='Skip recompiling if possible. Only recommended '
'for development purposes.')
parser.add_argument('-p',
'--pkg',
action='store_true',
help='Package the OpenSSL output directory.')
parser.add_argument('-i',
'--install',
action='store_true',
help='Install the OpenSSL package.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help="Increase verbosity level. Repeatable up to "
"2 times (-vv)")
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# set argument variables
log.verbose = args.verbose
skip = args.skip
root.root_check()
if args.build:
log.info("Bulding OpenSSL...")
check_dir = os.path.isdir(PKG_PAYLOAD_DIR)
# When the skip option is passed and the build directory exists, skip
# download and compiling of openssl. Note we still do linking.
if (skip and check_dir):
log.debug("Skip flag was provided. We will not compile OpenSSL "
"on this run.")
else:
download_and_extract_openssl()
build()
current_certs()
if args.pkg:
log.info("Building a package for OpenSSL...")
# Change back into our local directory so we can output our package
# via relative paths
os.chdir(CURRENT_DIR)
version = CONFIG['openssl_version']
rc = package.pkg(
root=PKG_PAYLOAD_DIR,
version=version,
identifier="{}.openssl".format(CONFIG['pkgid']),
output='openssl-{}.pkg'.format(version),
)
if rc == 0:
log.info("OpenSSL packaged properly")
else:
log.error("Looks like package creation failed")
if args.install:
log.info("Installing OpenSSL pacakge...")
os.chdir(CURRENT_DIR)
cmd = [
'/usr/sbin/installer', '-pkg', 'openssl-{}.pkg'.format(version),
'-tgt', '/'
]
runner.Popen(cmd)
def build():
"""Build our tlsssl patch."""
log.info("Building tlsssl...")
patch_dir = os.path.join(CURRENT_DIR, '_patch')
# Step 2: make sure the _ssl_data.h header has been generated
ssl_data = os.path.join(patch_dir, "_ssl_data.h")
if not os.path.isfile(ssl_data):
log.debug("Generate _ssl_data.h header...")
tool_path = os.path.join(CURRENT_DIR, "_patch", "make_ssl_data.py")
# Run the generating script
cmd = ['/usr/bin/python', tool_path, HEADER_SRC, ssl_data]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 3: remove the temporary work directory under the build dir
build_dir = os.path.join(CURRENT_DIR, 'build')
if os.path.exists(build_dir):
log.debug("Removing build directory...")
shutil.rmtree(build_dir, ignore_errors=True)
log.debug("Creating build directories...")
mkpath(build_dir)
# Step 3.5: copy ssl.py to the build directory
log.info("Copy 'ssl.py' to the build directory...")
shutil.copy(os.path.join(CURRENT_DIR, '_patch/ssl.py'), build_dir)
workspace_rel = os.path.join(build_dir)
workspace_abs = os.path.realpath(workspace_rel)
# Step 4: copy and rename the dylibs to there
log.detail("Copying dylibs to build directory")
ssl_src = os.path.join(LIBS_SRC, "libssl.dylib")
crypt_src = os.path.join(LIBS_SRC, "libcrypto.dylib")
ssl_tmp = os.path.join(workspace_abs, "libtlsssl.dylib")
crypt_tmp = os.path.join(workspace_abs, "libtlscrypto.dylib")
try:
shutil.copy(ssl_src, ssl_tmp)
shutil.copy(crypt_src, crypt_tmp)
except (IOError) as err:
log.warn("tlsssl has a dependency on OpenSSL 1.0.1+ as such you "
"must build and install OpenSSL from ../openssl.")
log.error("Build failed and will now exit!")
log.error("{}".format(err))
sys.exit(1)
# Step 5: change the ids of the dylibs
log.detail("Changing the ids of the dylibs...")
ssl_dest = os.path.join(LIBS_DEST, "libtlsssl.dylib")
crypt_dest = os.path.join(LIBS_DEST, "libtlscrypto.dylib")
# (need to temporarily mark them as writeable)
# NOTE: I don't think this I needed any longer
st = os.stat(ssl_tmp)
os.chmod(ssl_tmp, st.st_mode | stat.S_IWUSR)
st = os.stat(crypt_tmp)
os.chmod(crypt_tmp, st.st_mode | stat.S_IWUSR)
cmd = ['/usr/bin/install_name_tool', '-id', ssl_dest, ssl_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
cmd = ['/usr/bin/install_name_tool', '-id', crypt_dest, crypt_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 6: change the link between ssl and crypto
# This part is a bit trickier - we need to take the existing entry
# for libcrypto on libssl and remap it to the new location
cmd = ['/usr/bin/otool', '-L', ssl_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
old_path = re.findall('^\t(/[^\(]+?libcrypto.*?.dylib)', out[0],
re.MULTILINE)[0]
log.debug("The old path was: {}".format(old_path))
cmd = [
'/usr/bin/install_name_tool', '-change', old_path, crypt_dest, ssl_tmp
]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 7: cleanup permissions
# NOTE: Same. I don't think this I needed any longer
st = os.stat(ssl_tmp)
os.chmod(ssl_tmp, st.st_mode & ~stat.S_IWUSR)
st = os.stat(crypt_tmp)
os.chmod(crypt_tmp, st.st_mode & ~stat.S_IWUSR)
# Step 8: patch in the additional paths and linkages
# NOTE: This command will output a few warnings that are hidden at
# build time. Just an FYI in case this needs to be resolved in
# the future.
system_python_path = ("/System/Library/Frameworks/Python.framework/"
"Versions/2.7/include/python2.7")
cmd = [
"cc", "-fno-strict-aliasing", "-fno-common", "-dynamic", "-arch",
"x86_64", "-arch", "i386", "-g", "-Os", "-pipe", "-fno-common",
"-fno-strict-aliasing", "-fwrapv", "-DENABLE_DTRACE", "-DMACOSX",
"-DNDEBUG", "-Wall", "-Wstrict-prototypes", "-Wshorten-64-to-32",
"-DNDEBUG", "-g", "-fwrapv", "-Os", "-Wall", "-Wstrict-prototypes",
"-DENABLE_DTRACE", "-arch", "x86_64", "-arch", "i386", "-pipe",
"-I{}".format(HEADER_SRC), "-I{}".format(system_python_path), "-c",
"_patch/_ssl.c", "-o", "build/_ssl.o"
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Build of '_ssl.o' completed successfullyly")
else:
log.error("Build has failed: {}".format(out[1]))
cmd = [
"cc", "-bundle", "-undefined", "dynamic_lookup", "-arch", "x86_64",
"-arch", "i386", "-Wl,-F.", "build/_ssl.o",
"-L{}".format(workspace_abs), "-ltlsssl", "-ltlsssl", "-o",
"build/_ssl.so"
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Build of '_ssl.so' completed successfullyly")
else:
log.error("Build has failed: {}".format(out[1]))
log.debug("Remove temp '_ssl.o' from build directory")
os.remove(os.path.join(build_dir, "_ssl.o"))