def download_python_source_files():
"""Download CPython source files from Github.
Verify the sha hash and redownload if they do not match.
"""
log.info("Downloading and verifying python source files...")
src_dir = os.path.join(CURRENT_DIR, '_src')
if not os.path.exists(src_dir):
log.debug("Creating _src directory...")
mkpath(src_dir)
os.chdir(src_dir)
gh_url = 'https://raw.githubusercontent.com/python/cpython/v2.7.10/'
# This ugly looking block of code is a pair that matches the filename,
# github url, and sha256 hash for each required python source file
fp = [
['ssl.py', '{}Lib/ssl.py'.format(gh_url), CONFIG['ssl_py_hash']],
['_ssl.c', '{}Modules/_ssl.c'.format(gh_url), CONFIG['ssl_c_hash']],
[
'make_ssl_data.py', '{}Tools/ssl/make_ssl_data.py'.format(gh_url),
CONFIG['make_ssl_data_py_hash']
],
[
'socketmodule.h', '{}Modules/socketmodule.h'.format(gh_url),
CONFIG['socketmodule_h_hash']
],
]
# Verify we have the correct python source files else download it
log.detail("Downloading & checking hash of python source files...")
for fname, url, sha256 in fp:
# This is a dual check step for file existence and hash matching
log.debug("Checking source file: {}...".format(fname))
if not os.path.isfile(fname) or (hash_helper.getsha256hash(fname) !=
sha256):
log.info("Downloading '{}' source file...".format(fname))
log.debug("Download url: {}".format(url))
try:
data = urllib2.urlopen(url)
f = open(fname, "w")
content = data.read()
f.write(content)
f.close()
# Verify the hash of the source file we just downloaded
download_file_hash = hash_helper.getsha256hash(fname)
if download_file_hash != sha256:
log.warn("The hash for '{}' does not match the expected "
"hash. The downloaded hash is '{}'".format(
fname, download_file_hash))
else:
log.debug("The download file '{}' matches our expected "
"hash of '{}'".format(fname, sha256))
except (urllib2.HTTPError, urllib2.URLError, OSError,
IOError) as err:
log.error("Unable to download '{}' "
"due to {}\n".format(fname, err))
sys.exit(1)
# We are done with _src directory for now so go back to script root path
os.chdir(CURRENT_DIR)
def dl_and_extract_python(dist_url, dist_hash):
"""Download Python distribution and extract it to PYTHON_BUILD_DIR."""
if os.path.isdir(PYTHON_BUILD_DIR):
shutil.rmtree(PYTHON_BUILD_DIR, ignore_errors=True)
mkpath(PYTHON_BUILD_DIR)
# Download Python
log.info("Downloading Python from: {}".format(dist_url))
temp_filename = os.path.join(tempfile.mkdtemp(), 'tempdata')
cmd = [
'/usr/bin/curl', '--show-error', '--no-buffer', '--fail',
'--progress-bar', '--speed-time', '30', '--location', '--url',
dist_url, '--output', temp_filename
]
# We are calling os.system so we can get download progress live
rc = runner.system(cmd)
if rc == 0 or rc is True:
log.debug("Python download successful")
else:
log.error("Python download failed with exit code: '{}'".format(rc))
sys.exit(1)
# Verify Python download hash
download_hash = hash_helper.getsha256hash(temp_filename)
config_hash = dist_hash
if download_hash != config_hash:
log.error("Hash verification of Python download has failed. Download "
"hash of '{}' does not match config hash '{}'".format(
download_hash, config_hash))
sys.exit(1)
else:
log.detail("Hash verification of Python successful")
# Extract Python to the PYTHON_BUILD_DIR
log.info("Extracting Python...")
cmd = [
'/usr/bin/tar', '-xf', temp_filename, '-C', PYTHON_BUILD_DIR,
'--strip-components', '1'
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Extraction completed successfully")
else:
log.error("Extraction has failed: {}".format(out[0]))
os.remove(temp_filename)
def download_and_extract_openssl():
"""Download openssl distribution and extract it to OPENSSL_BUILD_DIR."""
if os.path.isdir(OPENSSL_BUILD_DIR):
shutil.rmtree(OPENSSL_BUILD_DIR, ignore_errors=True)
mkpath(OPENSSL_BUILD_DIR)
# Download openssl
log.info("Downloading OpenSSL from: {}".format(CONFIG['openssl_dist']))
temp_filename = os.path.join(tempfile.mkdtemp(), 'tempdata')
cmd = [
'/usr/bin/curl', '--show-error', '--no-buffer', '--fail',
'--progress-bar', '--speed-time', '30', '--location', '--url',
CONFIG['openssl_dist'], '--output', temp_filename
]
# We are calling os.system so we can get download progress live
rc = runner.system(cmd)
if rc == 0 or rc is True:
log.debug("OpenSSL download successfully")
else:
log.error("OpenSSL download failed with exit code: '{}'".format(rc))
sys.exit(1)
# Verify openssl download hash
download_hash = hash_helper.getsha256hash(temp_filename)
config_hash = CONFIG['openssl_dist_hash']
if download_hash != config_hash:
log.error("Hash verification of OpenSSL download has failed. Download "
"hash of '{}' does not match config hash '{}'".format(
download_hash, config_hash))
sys.exit(1)
else:
log.detail("Hash verification of OpenSSL successfully")
# Extract openssl to the openssl_build_dir
log.info("Extracting OpenSSL...")
cmd = [
'/usr/bin/tar', '-xf', temp_filename, '-C', OPENSSL_BUILD_DIR,
'--strip-components', '1'
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Extraction completed successfullyly")
else:
log.error("Extraction has failed: {}".format(out[1]))
os.remove(temp_filename)
def patch():
"""Patch source files for the build phase."""
log.info("Creating our patch files for tlsssl...")
patch_dir = os.path.join(CURRENT_DIR, '_patch')
if not os.path.exists(patch_dir):
log.debug("Creating _patch directory...")
mkpath(patch_dir)
patch_pairs = [
# ['_patch/_ssl.c', '_src/_ssl.c', ],
['_patch/make_ssl_data.py', '_src/make_ssl_data.py'],
# ['_patch/ssl.py', '_src/ssl.py'],
]
log.detail("Create our patch files if they do not exist...")
for dest, source in patch_pairs:
if not os.path.isfile(os.path.join(CURRENT_DIR, dest)):
source = os.path.join(CURRENT_DIR, source)
diff = os.path.join(CURRENT_DIR, '_diffs',
"{}.diff".format(os.path.basename(dest)))
dest = os.path.join(CURRENT_DIR, dest)
log.debug("Patching '{}'".format(dest))
# TODO: Validate the return code and exist if something didn't work
cmd = ['/usr/bin/patch', source, diff, "-o", dest]
out = runner.Popen(cmd)
runner.pprint(out)
# Copy over the socketmodule.h file as well
if not os.path.isfile(os.path.join(patch_dir, "socketmodule.h")):
log.debug("Copying 'socketmodule.h' to the _patch dir")
source = os.path.join(CURRENT_DIR, "_src", "socketmodule.h")
shutil.copy(source, os.path.realpath(os.path.join(patch_dir)))
if not os.path.isfile(os.path.join(patch_dir, "_ssl.c")):
log.debug("Copying '_ssl.c' to the _patch dir")
source = os.path.join(CURRENT_DIR, "_src", "_ssl.c")
shutil.copy(source, os.path.realpath(os.path.join(patch_dir)))
if not os.path.isfile(os.path.join(patch_dir, "ssl.py")):
log.debug("Copying 'ssl.py' to the _patch dir")
source = os.path.join(CURRENT_DIR, "_src", "ssl.py")
shutil.copy(source, os.path.realpath(os.path.join(patch_dir)))
log.detail("All patch files are created...")
def build(py_version, py_install_path, skip):
"""Build custom Python from source."""
py_major_ver = py_version.split('.')[0]
log.debug("Currently building: {}".format(py_major_ver))
# Step 1: change into our build directory
os.chdir(PYTHON_BUILD_DIR)
# Don't compile Python if the skip option is passed
if skip:
log.info("Python compile skipped due to -skip option")
return
# Step 1.5: Add extra modules
if py_major_ver == '2':
setup_dist = os.path.join(PYTHON_BUILD_DIR, 'Modules/Setup.dist')
with open(setup_dist, "a") as f:
log.debug("Adding additional modules to be included...")
f.write("_socket socketmodule.c timemodule.c\n")
f.write("_ssl _ssl.c -DUSE_SSL "
"-I{0}/include -I{0}/include/openssl -L{0}/lib "
"-lssl -lcrypto".format(OPENSSL_INSTALL_PATH))
# Step 2: Run the Configure setup of Python to set correct paths
os.chdir(PYTHON_BUILD_DIR)
if os.path.isdir(py_install_path):
shutil.rmtree(py_install_path, ignore_errors=True)
mkpath(py_install_path)
log.info("Configuring Python...")
cmd = [
'./configure',
'--prefix={}'.format(py_install_path),
# 'CPPFLAGS=-I{}/include'.format(OPENSSL_INSTALL_PATH),
# 'LDFLAGS=-L{}/lib'.format(OPENSSL_INSTALL_PATH),
'CFLAGS=-I{}/include'.format(OPENSSL_INSTALL_PATH),
'LDFLAGS=-L{}/lib'.format(OPENSSL_INSTALL_PATH),
'--enable-shared',
'--enable-toolbox-glue',
'--with-ensurepip=install',
'--enable-ipv6',
'--with-threads',
'--datarootdir={}/share'.format(py_install_path),
'--datadir={}/share'.format(py_install_path),
# '--enable-optimizations', # adding this flag will run tests
]
runner.Popen(cmd, stdout=sys.stdout)
# Step 3: compile Python. this will take a while.
# FIXME: We need to check return codes.
log.info("Compiling Python. This will take a while time...")
log.detail("Running Python make routine...")
cmd = ['/usr/bin/make']
runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
log.debug("Create some temp files thats")
log.detail("Running Python make install routine...")
cmd = ['/usr/bin/make', 'install']
runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
# Step 4: Install pip + requirements
os.chdir(os.path.join(py_install_path, 'bin'))
# Update pip to latest
log.info("Upgrading pip...")
if py_major_ver == '2':
cmd = ['./pip']
elif py_major_ver == '3':
cmd = ['./pip3']
cmd = cmd + ['install', '--upgrade', 'pip']
runner.Popen(cmd, stdout=sys.stdout)
# Install all pip modules from requirements.txt
log.info("Install requirements...")
if py_major_ver == '2':
cmd = [
'./python2.7', '-m', 'pip', 'install', '-r',
os.path.join(CURRENT_DIR, 'requirements2.txt')
]
elif py_major_ver == '3':
cmd = [
'./python3.6', '-m', 'pip', 'install', '-r',
os.path.join(CURRENT_DIR, 'requirements3.txt')
]
runner.Popen(cmd, stdout=sys.stdout)
def build():
"""Build OpenSSL from source."""
# Step 1: change into our build directory
os.chdir(OPENSSL_BUILD_DIR)
# Don't compile openssl if the skip option is passed
# Step 2: Run the Configure setup of OpenSSL to set correct paths
openssl_install = os.path.join(BASE_INSTALL_PATH, 'openssl')
log.info("Configuring OpenSSL...")
cmd = [
'./Configure',
'--prefix={}'.format(openssl_install),
'--openssldir={}'.format(openssl_install),
'darwin64-x86_64-cc',
'enable-ec_nistp_64_gcc_128',
'no-ssl2',
'no-ssl3',
'no-zlib',
'shared',
'enable-cms',
'no-comp',
]
# OpenSSL 1.0 to 1.1 has some pretty major API and build differences.
# Manage the build control with the following. NOTE: 1.1 is not
# supported at this time. Hopefully in a future release.
OLD_VERSION = None
# OpenSSL renamed this build flag so was less confusing
# https://github.com/openssl/openssl/commit/3c65577f1af1109beb8de06420efa09188981628
TMP_DIR_FLAG = None
if OPENSSL_VERSION > "1.1.0":
OLD_VERSION = False
TMP_DIR_FLAG = "DESTDIR"
else:
OLD_VERSION = True
TMP_DIR_FLAG = "INSTALL_PREFIX"
# If running 1.0 use runner.system() else runner.Popen()
if OLD_VERSION:
out = runner.system(cmd)
else:
out = runner.Popen(cmd)
log.debug("Configuring returned value: {}".format(out))
# Step 3: compile openssl. this will take a while.
# FIXME: We need to check return codes.
# This command is required for OpenSSL lower than 1.1
log.info("Compiling OpenSSL. This will take a while time...")
if OLD_VERSION:
log.detail("Running OpenSSL make depend routine...")
cmd = ['/usr/bin/make', 'depend']
proc = subprocess.Popen(cmd, bufsize=-1, stdout=sys.stdout)
(output, dummy_error) = proc.communicate()
sys.stdout.flush() # does this help?
log.detail("Running OpenSSL make routine...")
cmd = ['/usr/bin/make']
proc = subprocess.Popen(cmd, bufsize=-1, stdout=sys.stdout)
(output, dummy_error) = proc.communicate()
sys.stdout.flush() # does this help?
# log.detail("Running OpenSSL make test routine...")
# cmd = ['/usr/bin/make', 'test']
# proc = subprocess.Popen(cmd, bufsize=-1, stdout=sys.stdout)
# (output, dummy_error) = proc.communicate()
# sys.stdout.flush() # does this help?
mkpath(PKG_PAYLOAD_DIR)
log.detail("Running OpenSSL make install routine...")
cmd = [
'/usr/bin/make', '{}={}'.format(TMP_DIR_FLAG, PKG_PAYLOAD_DIR),
'install'
]
print("ran a command: {}".format(' '.join(cmd)))
out = runner.Popen(cmd, stdout=sys.stdout)
sys.stdout.flush() # does this help?
def main():
"""Build and package the tlsssl patch."""
parser = argparse.ArgumentParser(prog='tlsssl setup',
description='This script will compile '
'tlsssl and optionally create '
'a native macOS package.')
parser.add_argument('-b',
'--build',
action='store_true',
help='Compile the tlsssl binaries')
parser.add_argument('-p',
'--pkg',
action='store_true',
help='Package the tlsssl output directory.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help="Increase verbosity level. Repeatable up to "
"2 times (-vv)")
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# set argument variables
log.verbose = args.verbose
if args.build:
log.info("Bulding tslssl...")
download_python_source_files()
patch()
build()
if args.pkg:
# FIXME: This has grown out of control. Move this outside of main!
log.info("Building a package for tlsssl...")
version = CONFIG['tlsssl_version']
# we need to setup the payload
payload_dir = os.path.join(CURRENT_DIR, 'payload')
if os.path.exists(payload_dir):
log.debug("Removing payload directory...")
shutil.rmtree(payload_dir, ignore_errors=True)
log.debug("Creating payload directory...")
payload_lib_dir = os.path.join(payload_dir, LIBS_DEST.lstrip('/'))
payload_root_dir = os.path.join(
payload_dir, CONFIG['tlsssl_install_dir'].lstrip('/'))
mkpath(payload_lib_dir)
log.detail("Changing file permissions for 'ssl.py'...")
# ssl.py needs to have chmod 644 so non-root users can import this
os.chmod('build/ssl.py',
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
log.detail("Copying build files into payload directory")
shutil.copy('build/_ssl.so', payload_root_dir)
shutil.copy('build/ssl.py', payload_root_dir)
shutil.copy('build/libtlscrypto.dylib', payload_lib_dir)
shutil.copy('build/libtlsssl.dylib', payload_lib_dir)
pth_fname = CONFIG['pth_fname']
# if the pth_fname key is set write the .pth file
if pth_fname is not '':
log.debug("Write the '.pth' file so native python can read "
"this module without a sys.path.insert")
python_sys = "/Library/Python/2.7/site-packages/"
python_sys_local = os.path.join("payload", python_sys.lstrip('/'))
log.debug("Make site-packages inside of payload")
mkpath(python_sys_local)
pth_file = os.path.join(python_sys_local, pth_fname)
f = open(pth_file, 'w')
# this hacky method will force this path to have a high priority
# http://stackoverflow.com/a/37380306
content = ("""import sys; sys.__plen = len(sys.path)
{}
import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; \
p=getattr(sys,'__egginsert',0); \
sys.path[p:p]=new; sys.__egginsert = p+len(new)""".format(
os.path.dirname(LIBS_DEST)))
f.write(content)
f.close()
rc = package.pkg(
root='payload',
version=version,
identifier="{}.tlsssl".format(CONFIG['pkgid']),
output='tlsssl-{}.pkg'.format(version),
)
if rc == 0:
log.info("tlsssl packaged properly")
else:
log.error("Looks like package creation failed")
def build():
"""Build our tlsssl patch."""
log.info("Building tlsssl...")
patch_dir = os.path.join(CURRENT_DIR, '_patch')
# Step 2: make sure the _ssl_data.h header has been generated
ssl_data = os.path.join(patch_dir, "_ssl_data.h")
if not os.path.isfile(ssl_data):
log.debug("Generate _ssl_data.h header...")
tool_path = os.path.join(CURRENT_DIR, "_patch", "make_ssl_data.py")
# Run the generating script
cmd = ['/usr/bin/python', tool_path, HEADER_SRC, ssl_data]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 3: remove the temporary work directory under the build dir
build_dir = os.path.join(CURRENT_DIR, 'build')
if os.path.exists(build_dir):
log.debug("Removing build directory...")
shutil.rmtree(build_dir, ignore_errors=True)
log.debug("Creating build directories...")
mkpath(build_dir)
# Step 3.5: copy ssl.py to the build directory
log.info("Copy 'ssl.py' to the build directory...")
shutil.copy(os.path.join(CURRENT_DIR, '_patch/ssl.py'), build_dir)
workspace_rel = os.path.join(build_dir)
workspace_abs = os.path.realpath(workspace_rel)
# Step 4: copy and rename the dylibs to there
log.detail("Copying dylibs to build directory")
ssl_src = os.path.join(LIBS_SRC, "libssl.dylib")
crypt_src = os.path.join(LIBS_SRC, "libcrypto.dylib")
ssl_tmp = os.path.join(workspace_abs, "libtlsssl.dylib")
crypt_tmp = os.path.join(workspace_abs, "libtlscrypto.dylib")
try:
shutil.copy(ssl_src, ssl_tmp)
shutil.copy(crypt_src, crypt_tmp)
except (IOError) as err:
log.warn("tlsssl has a dependency on OpenSSL 1.0.1+ as such you "
"must build and install OpenSSL from ../openssl.")
log.error("Build failed and will now exit!")
log.error("{}".format(err))
sys.exit(1)
# Step 5: change the ids of the dylibs
log.detail("Changing the ids of the dylibs...")
ssl_dest = os.path.join(LIBS_DEST, "libtlsssl.dylib")
crypt_dest = os.path.join(LIBS_DEST, "libtlscrypto.dylib")
# (need to temporarily mark them as writeable)
# NOTE: I don't think this I needed any longer
st = os.stat(ssl_tmp)
os.chmod(ssl_tmp, st.st_mode | stat.S_IWUSR)
st = os.stat(crypt_tmp)
os.chmod(crypt_tmp, st.st_mode | stat.S_IWUSR)
cmd = ['/usr/bin/install_name_tool', '-id', ssl_dest, ssl_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
cmd = ['/usr/bin/install_name_tool', '-id', crypt_dest, crypt_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 6: change the link between ssl and crypto
# This part is a bit trickier - we need to take the existing entry
# for libcrypto on libssl and remap it to the new location
cmd = ['/usr/bin/otool', '-L', ssl_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
old_path = re.findall('^\t(/[^\(]+?libcrypto.*?.dylib)', out[0],
re.MULTILINE)[0]
log.debug("The old path was: {}".format(old_path))
cmd = [
'/usr/bin/install_name_tool', '-change', old_path, crypt_dest, ssl_tmp
]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 7: cleanup permissions
# NOTE: Same. I don't think this I needed any longer
st = os.stat(ssl_tmp)
os.chmod(ssl_tmp, st.st_mode & ~stat.S_IWUSR)
st = os.stat(crypt_tmp)
os.chmod(crypt_tmp, st.st_mode & ~stat.S_IWUSR)
# Step 8: patch in the additional paths and linkages
# NOTE: This command will output a few warnings that are hidden at
# build time. Just an FYI in case this needs to be resolved in
# the future.
system_python_path = ("/System/Library/Frameworks/Python.framework/"
"Versions/2.7/include/python2.7")
cmd = [
"cc", "-fno-strict-aliasing", "-fno-common", "-dynamic", "-arch",
"x86_64", "-arch", "i386", "-g", "-Os", "-pipe", "-fno-common",
"-fno-strict-aliasing", "-fwrapv", "-DENABLE_DTRACE", "-DMACOSX",
"-DNDEBUG", "-Wall", "-Wstrict-prototypes", "-Wshorten-64-to-32",
"-DNDEBUG", "-g", "-fwrapv", "-Os", "-Wall", "-Wstrict-prototypes",
"-DENABLE_DTRACE", "-arch", "x86_64", "-arch", "i386", "-pipe",
"-I{}".format(HEADER_SRC), "-I{}".format(system_python_path), "-c",
"_patch/_ssl.c", "-o", "build/_ssl.o"
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Build of '_ssl.o' completed successfullyly")
else:
log.error("Build has failed: {}".format(out[1]))
cmd = [
"cc", "-bundle", "-undefined", "dynamic_lookup", "-arch", "x86_64",
"-arch", "i386", "-Wl,-F.", "build/_ssl.o",
"-L{}".format(workspace_abs), "-ltlsssl", "-ltlsssl", "-o",
"build/_ssl.so"
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Build of '_ssl.so' completed successfullyly")
else:
log.error("Build has failed: {}".format(out[1]))
log.debug("Remove temp '_ssl.o' from build directory")
os.remove(os.path.join(build_dir, "_ssl.o"))