def download_python_source_files():
"""Download CPython source files from Github.
Verify the sha hash and redownload if they do not match.
"""
log.info("Downloading and verifying python source files...")
src_dir = os.path.join(CURRENT_DIR, '_src')
if not os.path.exists(src_dir):
log.debug("Creating _src directory...")
mkpath(src_dir)
os.chdir(src_dir)
gh_url = 'https://raw.githubusercontent.com/python/cpython/v2.7.10/'
# This ugly looking block of code is a pair that matches the filename,
# github url, and sha256 hash for each required python source file
fp = [
['ssl.py', '{}Lib/ssl.py'.format(gh_url), CONFIG['ssl_py_hash']],
['_ssl.c', '{}Modules/_ssl.c'.format(gh_url), CONFIG['ssl_c_hash']],
[
'make_ssl_data.py', '{}Tools/ssl/make_ssl_data.py'.format(gh_url),
CONFIG['make_ssl_data_py_hash']
],
[
'socketmodule.h', '{}Modules/socketmodule.h'.format(gh_url),
CONFIG['socketmodule_h_hash']
],
]
# Verify we have the correct python source files else download it
log.detail("Downloading & checking hash of python source files...")
for fname, url, sha256 in fp:
# This is a dual check step for file existence and hash matching
log.debug("Checking source file: {}...".format(fname))
if not os.path.isfile(fname) or (hash_helper.getsha256hash(fname) !=
sha256):
log.info("Downloading '{}' source file...".format(fname))
log.debug("Download url: {}".format(url))
try:
data = urllib2.urlopen(url)
f = open(fname, "w")
content = data.read()
f.write(content)
f.close()
# Verify the hash of the source file we just downloaded
download_file_hash = hash_helper.getsha256hash(fname)
if download_file_hash != sha256:
log.warn("The hash for '{}' does not match the expected "
"hash. The downloaded hash is '{}'".format(
fname, download_file_hash))
else:
log.debug("The download file '{}' matches our expected "
"hash of '{}'".format(fname, sha256))
except (urllib2.HTTPError, urllib2.URLError, OSError,
IOError) as err:
log.error("Unable to download '{}' "
"due to {}\n".format(fname, err))
sys.exit(1)
# We are done with _src directory for now so go back to script root path
os.chdir(CURRENT_DIR)
def current_certs():
"""
Include current SystemRoot certs with OpenSSL.
This helps work around a limitation with bundling your own version of
OpenSSL. We copy the certs from Apple's 'SystemRootCertificates.keychain'
into OPENSSL_BUILD_DIR/cert.pem
https://goo.gl/s6vvwl
"""
log.info("Writing the 'cert.pem' file from Apple's System Root Certs...")
cmd = [
'/usr/bin/security', 'find-certificate', '-a', '-p',
'/System/Library/Keychains/SystemRootCertificates.keychain'
]
out = runner.Popen(cmd)
# Take the command output, out[0], and write the file to cert.pem
cert_path = os.path.join(PKG_PAYLOAD_DIR, BASE_INSTALL_PATH_S, 'openssl',
'cert.pem')
try:
f = open(cert_path, "w")
f.write(out[0])
f.close()
except (IOError) as e:
log.error("Unable to write 'cert.pem': {}".format(e))
def dl_and_extract_python(dist_url, dist_hash):
"""Download Python distribution and extract it to PYTHON_BUILD_DIR."""
if os.path.isdir(PYTHON_BUILD_DIR):
shutil.rmtree(PYTHON_BUILD_DIR, ignore_errors=True)
mkpath(PYTHON_BUILD_DIR)
# Download Python
log.info("Downloading Python from: {}".format(dist_url))
temp_filename = os.path.join(tempfile.mkdtemp(), 'tempdata')
cmd = [
'/usr/bin/curl', '--show-error', '--no-buffer', '--fail',
'--progress-bar', '--speed-time', '30', '--location', '--url',
dist_url, '--output', temp_filename
]
# We are calling os.system so we can get download progress live
rc = runner.system(cmd)
if rc == 0 or rc is True:
log.debug("Python download successful")
else:
log.error("Python download failed with exit code: '{}'".format(rc))
sys.exit(1)
# Verify Python download hash
download_hash = hash_helper.getsha256hash(temp_filename)
config_hash = dist_hash
if download_hash != config_hash:
log.error("Hash verification of Python download has failed. Download "
"hash of '{}' does not match config hash '{}'".format(
download_hash, config_hash))
sys.exit(1)
else:
log.detail("Hash verification of Python successful")
# Extract Python to the PYTHON_BUILD_DIR
log.info("Extracting Python...")
cmd = [
'/usr/bin/tar', '-xf', temp_filename, '-C', PYTHON_BUILD_DIR,
'--strip-components', '1'
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Extraction completed successfully")
else:
log.error("Extraction has failed: {}".format(out[0]))
os.remove(temp_filename)
def download_and_extract_openssl():
"""Download openssl distribution and extract it to OPENSSL_BUILD_DIR."""
if os.path.isdir(OPENSSL_BUILD_DIR):
shutil.rmtree(OPENSSL_BUILD_DIR, ignore_errors=True)
mkpath(OPENSSL_BUILD_DIR)
# Download openssl
log.info("Downloading OpenSSL from: {}".format(CONFIG['openssl_dist']))
temp_filename = os.path.join(tempfile.mkdtemp(), 'tempdata')
cmd = [
'/usr/bin/curl', '--show-error', '--no-buffer', '--fail',
'--progress-bar', '--speed-time', '30', '--location', '--url',
CONFIG['openssl_dist'], '--output', temp_filename
]
# We are calling os.system so we can get download progress live
rc = runner.system(cmd)
if rc == 0 or rc is True:
log.debug("OpenSSL download successfully")
else:
log.error("OpenSSL download failed with exit code: '{}'".format(rc))
sys.exit(1)
# Verify openssl download hash
download_hash = hash_helper.getsha256hash(temp_filename)
config_hash = CONFIG['openssl_dist_hash']
if download_hash != config_hash:
log.error("Hash verification of OpenSSL download has failed. Download "
"hash of '{}' does not match config hash '{}'".format(
download_hash, config_hash))
sys.exit(1)
else:
log.detail("Hash verification of OpenSSL successfully")
# Extract openssl to the openssl_build_dir
log.info("Extracting OpenSSL...")
cmd = [
'/usr/bin/tar', '-xf', temp_filename, '-C', OPENSSL_BUILD_DIR,
'--strip-components', '1'
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Extraction completed successfullyly")
else:
log.error("Extraction has failed: {}".format(out[1]))
os.remove(temp_filename)
def main():
"""Build and package Python2."""
parser = argparse.ArgumentParser(prog='Python setup',
description='This script will compile '
'Python 1.0.1+ and optionally create '
'a native macOS package.')
parser.add_argument('-b',
'--build',
action='store_true',
help='Compile the Python binary')
parser.add_argument('-s',
'--skip',
action='store_true',
help='Skip recompiling if possible. Only recommended '
'for development purposes.')
parser.add_argument('-p',
'--pkg',
action='store_true',
help='Package the Python output directory.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help="Increase verbosity level. Repeatable up to "
"2 times (-vv)")
parser.add_argument('--py',
default='2',
help='Python version to build. Accepts 2 or 3.')
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# set argument variables
log.verbose = args.verbose
skip = args.skip
root.root_check()
# Check for OpenSSL. If it isn't on disk in the proper location
# we can't link against it.
if not os.path.isdir(OPENSSL_INSTALL_PATH):
log.warn("OpenSSL must be installed to '{}' prior to compiling "
"Python.".format(OPENSSL_INSTALL_PATH))
sys.exit(1)
if str(args.py) == '2':
dist_url = CONFIG['python2_dist']
dist_hash = CONFIG['python2_dist_hash']
py_install_path = PYTHON2_INSTALL
py_version = PYTHON2_VERSION
elif str(args.py) == '3':
dist_url = CONFIG['python3_dist']
dist_hash = CONFIG['python3_dist_hash']
py_install_path = PYTHON3_INSTALL
py_version = PYTHON3_VERSION
else:
sys.stderr('Unsupported python version\n')
sys.exit(1)
if args.build:
log.info("Bulding Python...")
# When the skip option is passed and the build directory exists, skip
# download and compiling of Python. Note we still do linking.
if skip:
log.debug("Skip flag was provided. We will not compile Python "
"on this run.")
else:
dl_and_extract_python(dist_url, dist_hash)
# reset trigger flag as we needed to download Python
skip = False
build(py_version, py_install_path, skip=skip)
if args.pkg:
log.info("Building a package for Python...")
# Change back into our local directory so we can output our package
# via relative paths
os.chdir(CURRENT_DIR)
rc = package.pkg(
root=py_install_path,
version=py_version,
identifier="{}.python".format(CONFIG['pkgid']),
install_location=py_install_path,
output='python-{}.pkg'.format(py_version),
)
if rc == 0:
log.info("Python packaged properly")
else:
log.error("Looks like package creation failed")
def main():
"""Build and package OpenSSL."""
parser = argparse.ArgumentParser(prog='OpenSSL setup',
description='This script will compile '
'OpenSSL 1.0.1+ and optionally create '
'a native macOS package.')
parser.add_argument('-b',
'--build',
action='store_true',
help='Compile the OpenSSL binary')
parser.add_argument('-s',
'--skip',
action='store_true',
help='Skip recompiling if possible. Only recommended '
'for development purposes.')
parser.add_argument('-p',
'--pkg',
action='store_true',
help='Package the OpenSSL output directory.')
parser.add_argument('-i',
'--install',
action='store_true',
help='Install the OpenSSL package.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help="Increase verbosity level. Repeatable up to "
"2 times (-vv)")
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# set argument variables
log.verbose = args.verbose
skip = args.skip
root.root_check()
if args.build:
log.info("Bulding OpenSSL...")
check_dir = os.path.isdir(PKG_PAYLOAD_DIR)
# When the skip option is passed and the build directory exists, skip
# download and compiling of openssl. Note we still do linking.
if (skip and check_dir):
log.debug("Skip flag was provided. We will not compile OpenSSL "
"on this run.")
else:
download_and_extract_openssl()
build()
current_certs()
if args.pkg:
log.info("Building a package for OpenSSL...")
# Change back into our local directory so we can output our package
# via relative paths
os.chdir(CURRENT_DIR)
version = CONFIG['openssl_version']
rc = package.pkg(
root=PKG_PAYLOAD_DIR,
version=version,
identifier="{}.openssl".format(CONFIG['pkgid']),
output='openssl-{}.pkg'.format(version),
)
if rc == 0:
log.info("OpenSSL packaged properly")
else:
log.error("Looks like package creation failed")
if args.install:
log.info("Installing OpenSSL pacakge...")
os.chdir(CURRENT_DIR)
cmd = [
'/usr/sbin/installer', '-pkg', 'openssl-{}.pkg'.format(version),
'-tgt', '/'
]
runner.Popen(cmd)
def main():
"""Build and package the tlsssl patch."""
parser = argparse.ArgumentParser(prog='tlsssl setup',
description='This script will compile '
'tlsssl and optionally create '
'a native macOS package.')
parser.add_argument('-b',
'--build',
action='store_true',
help='Compile the tlsssl binaries')
parser.add_argument('-p',
'--pkg',
action='store_true',
help='Package the tlsssl output directory.')
parser.add_argument('-v',
'--verbose',
action='count',
default=1,
help="Increase verbosity level. Repeatable up to "
"2 times (-vv)")
if len(sys.argv) < 2:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
# set argument variables
log.verbose = args.verbose
if args.build:
log.info("Bulding tslssl...")
download_python_source_files()
patch()
build()
if args.pkg:
# FIXME: This has grown out of control. Move this outside of main!
log.info("Building a package for tlsssl...")
version = CONFIG['tlsssl_version']
# we need to setup the payload
payload_dir = os.path.join(CURRENT_DIR, 'payload')
if os.path.exists(payload_dir):
log.debug("Removing payload directory...")
shutil.rmtree(payload_dir, ignore_errors=True)
log.debug("Creating payload directory...")
payload_lib_dir = os.path.join(payload_dir, LIBS_DEST.lstrip('/'))
payload_root_dir = os.path.join(
payload_dir, CONFIG['tlsssl_install_dir'].lstrip('/'))
mkpath(payload_lib_dir)
log.detail("Changing file permissions for 'ssl.py'...")
# ssl.py needs to have chmod 644 so non-root users can import this
os.chmod('build/ssl.py',
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
log.detail("Copying build files into payload directory")
shutil.copy('build/_ssl.so', payload_root_dir)
shutil.copy('build/ssl.py', payload_root_dir)
shutil.copy('build/libtlscrypto.dylib', payload_lib_dir)
shutil.copy('build/libtlsssl.dylib', payload_lib_dir)
pth_fname = CONFIG['pth_fname']
# if the pth_fname key is set write the .pth file
if pth_fname is not '':
log.debug("Write the '.pth' file so native python can read "
"this module without a sys.path.insert")
python_sys = "/Library/Python/2.7/site-packages/"
python_sys_local = os.path.join("payload", python_sys.lstrip('/'))
log.debug("Make site-packages inside of payload")
mkpath(python_sys_local)
pth_file = os.path.join(python_sys_local, pth_fname)
f = open(pth_file, 'w')
# this hacky method will force this path to have a high priority
# http://stackoverflow.com/a/37380306
content = ("""import sys; sys.__plen = len(sys.path)
{}
import sys; new=sys.path[sys.__plen:]; del sys.path[sys.__plen:]; \
p=getattr(sys,'__egginsert',0); \
sys.path[p:p]=new; sys.__egginsert = p+len(new)""".format(
os.path.dirname(LIBS_DEST)))
f.write(content)
f.close()
rc = package.pkg(
root='payload',
version=version,
identifier="{}.tlsssl".format(CONFIG['pkgid']),
output='tlsssl-{}.pkg'.format(version),
)
if rc == 0:
log.info("tlsssl packaged properly")
else:
log.error("Looks like package creation failed")
def build():
"""Build our tlsssl patch."""
log.info("Building tlsssl...")
patch_dir = os.path.join(CURRENT_DIR, '_patch')
# Step 2: make sure the _ssl_data.h header has been generated
ssl_data = os.path.join(patch_dir, "_ssl_data.h")
if not os.path.isfile(ssl_data):
log.debug("Generate _ssl_data.h header...")
tool_path = os.path.join(CURRENT_DIR, "_patch", "make_ssl_data.py")
# Run the generating script
cmd = ['/usr/bin/python', tool_path, HEADER_SRC, ssl_data]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 3: remove the temporary work directory under the build dir
build_dir = os.path.join(CURRENT_DIR, 'build')
if os.path.exists(build_dir):
log.debug("Removing build directory...")
shutil.rmtree(build_dir, ignore_errors=True)
log.debug("Creating build directories...")
mkpath(build_dir)
# Step 3.5: copy ssl.py to the build directory
log.info("Copy 'ssl.py' to the build directory...")
shutil.copy(os.path.join(CURRENT_DIR, '_patch/ssl.py'), build_dir)
workspace_rel = os.path.join(build_dir)
workspace_abs = os.path.realpath(workspace_rel)
# Step 4: copy and rename the dylibs to there
log.detail("Copying dylibs to build directory")
ssl_src = os.path.join(LIBS_SRC, "libssl.dylib")
crypt_src = os.path.join(LIBS_SRC, "libcrypto.dylib")
ssl_tmp = os.path.join(workspace_abs, "libtlsssl.dylib")
crypt_tmp = os.path.join(workspace_abs, "libtlscrypto.dylib")
try:
shutil.copy(ssl_src, ssl_tmp)
shutil.copy(crypt_src, crypt_tmp)
except (IOError) as err:
log.warn("tlsssl has a dependency on OpenSSL 1.0.1+ as such you "
"must build and install OpenSSL from ../openssl.")
log.error("Build failed and will now exit!")
log.error("{}".format(err))
sys.exit(1)
# Step 5: change the ids of the dylibs
log.detail("Changing the ids of the dylibs...")
ssl_dest = os.path.join(LIBS_DEST, "libtlsssl.dylib")
crypt_dest = os.path.join(LIBS_DEST, "libtlscrypto.dylib")
# (need to temporarily mark them as writeable)
# NOTE: I don't think this I needed any longer
st = os.stat(ssl_tmp)
os.chmod(ssl_tmp, st.st_mode | stat.S_IWUSR)
st = os.stat(crypt_tmp)
os.chmod(crypt_tmp, st.st_mode | stat.S_IWUSR)
cmd = ['/usr/bin/install_name_tool', '-id', ssl_dest, ssl_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
cmd = ['/usr/bin/install_name_tool', '-id', crypt_dest, crypt_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 6: change the link between ssl and crypto
# This part is a bit trickier - we need to take the existing entry
# for libcrypto on libssl and remap it to the new location
cmd = ['/usr/bin/otool', '-L', ssl_tmp]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
old_path = re.findall('^\t(/[^\(]+?libcrypto.*?.dylib)', out[0],
re.MULTILINE)[0]
log.debug("The old path was: {}".format(old_path))
cmd = [
'/usr/bin/install_name_tool', '-change', old_path, crypt_dest, ssl_tmp
]
out = runner.Popen(cmd)
runner.pprint(out, 'debug')
# Step 7: cleanup permissions
# NOTE: Same. I don't think this I needed any longer
st = os.stat(ssl_tmp)
os.chmod(ssl_tmp, st.st_mode & ~stat.S_IWUSR)
st = os.stat(crypt_tmp)
os.chmod(crypt_tmp, st.st_mode & ~stat.S_IWUSR)
# Step 8: patch in the additional paths and linkages
# NOTE: This command will output a few warnings that are hidden at
# build time. Just an FYI in case this needs to be resolved in
# the future.
system_python_path = ("/System/Library/Frameworks/Python.framework/"
"Versions/2.7/include/python2.7")
cmd = [
"cc", "-fno-strict-aliasing", "-fno-common", "-dynamic", "-arch",
"x86_64", "-arch", "i386", "-g", "-Os", "-pipe", "-fno-common",
"-fno-strict-aliasing", "-fwrapv", "-DENABLE_DTRACE", "-DMACOSX",
"-DNDEBUG", "-Wall", "-Wstrict-prototypes", "-Wshorten-64-to-32",
"-DNDEBUG", "-g", "-fwrapv", "-Os", "-Wall", "-Wstrict-prototypes",
"-DENABLE_DTRACE", "-arch", "x86_64", "-arch", "i386", "-pipe",
"-I{}".format(HEADER_SRC), "-I{}".format(system_python_path), "-c",
"_patch/_ssl.c", "-o", "build/_ssl.o"
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Build of '_ssl.o' completed successfullyly")
else:
log.error("Build has failed: {}".format(out[1]))
cmd = [
"cc", "-bundle", "-undefined", "dynamic_lookup", "-arch", "x86_64",
"-arch", "i386", "-Wl,-F.", "build/_ssl.o",
"-L{}".format(workspace_abs), "-ltlsssl", "-ltlsssl", "-o",
"build/_ssl.so"
]
out = runner.Popen(cmd)
if out[2] == 0:
log.debug("Build of '_ssl.so' completed successfullyly")
else:
log.error("Build has failed: {}".format(out[1]))
log.debug("Remove temp '_ssl.o' from build directory")
os.remove(os.path.join(build_dir, "_ssl.o"))