def _get_test_metadata(self):
f = NamedTemporaryFile()
f.write(ALL_TESTS_JSON)
f.flush()
self._temp_files.append(f)
return TestMetadata(filename=f.name)
def createCertificateAuthority(build, srcDir):
certutil = build.get_binary_path(what="certutil")
pk12util = build.get_binary_path(what="pk12util")
#TODO: mozfile.TemporaryDirectory
tempDbDir = tempfile.mkdtemp()
with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
pgoCAModulePathSrc = os.path.join(srcDir, "pgoca.p12")
pgoCAPathSrc = os.path.join(srcDir, "pgoca.ca")
pwfile.write("\n")
# Create temporary certification database for CA generation
status = runUtil(certutil, ["-N", "-d", tempDbDir, "-f", pwfile.name])
if status:
return status
createRandomFile(rndfile)
status = runUtil(certutil, [
"-S", "-d", tempDbDir, "-s",
"CN=Temporary Certificate Authority, O=Mozilla Testing, OU=Profile Guided Optimization",
"-t", "C,,", "-x", "-m", "1", "-v", "120", "-n",
"pgo temporary ca", "-2", "-f", pwfile.name, "-z", rndfile.name
], "Y\n0\nN\n")
if status:
return status
status = runUtil(certutil, [
"-L", "-d", tempDbDir, "-n", "pgo temporary ca", "-a", "-o",
pgoCAPathSrc, "-f", pwfile.name
])
if status:
return status
status = runUtil(pk12util, [
"-o", pgoCAModulePathSrc, "-n", "pgo temporary ca", "-d",
tempDbDir, "-w", pwfile.name, "-k", pwfile.name
])
if status:
return status
shutil.rmtree(tempDbDir)
return 0
def _get_test_metadata(self):
all_tests = NamedTemporaryFile(mode='wb')
pickle.dump(ALL_TESTS, all_tests)
all_tests.flush()
self._temp_files.append(all_tests)
test_defaults = NamedTemporaryFile(mode='wb')
pickle.dump(TEST_DEFAULTS, test_defaults)
test_defaults.flush()
self._temp_files.append(test_defaults)
return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
def vendor(self, packages=None, with_windows_wheel=False):
self.populate_logger()
self.log_manager.enable_unstructured()
vendor_dir = mozpath.join(self.topsrcdir,
os.path.join('third_party', 'python'))
packages = packages or []
if with_windows_wheel and len(packages) != 1:
raise Exception(
'--with-windows-wheel is only supported for a single package!')
self._activate_virtualenv()
pip_compile = os.path.join(self.virtualenv_manager.bin_path,
'pip-compile')
if not os.path.exists(pip_compile):
path = os.path.normpath(
os.path.join(self.topsrcdir, 'third_party', 'python',
'pip-tools'))
self.virtualenv_manager.install_pip_package(path, vendored=True)
spec = os.path.join(vendor_dir, 'requirements.in')
requirements = os.path.join(vendor_dir, 'requirements.txt')
with NamedTemporaryFile('w') as tmpspec:
shutil.copyfile(spec, tmpspec.name)
self._update_packages(tmpspec.name, packages)
# resolve the dependencies and update requirements.txt
subprocess.check_output([
pip_compile, tmpspec.name, '--no-header', '--no-index',
'--output-file', requirements, '--generate-hashes'
])
with TemporaryDirectory() as tmp:
# use requirements.txt to download archived source distributions of all packages
self.virtualenv_manager._run_pip([
'download', '-r', requirements, '--no-deps', '--dest', tmp,
'--no-binary', ':all:', '--disable-pip-version-check'
])
if with_windows_wheel:
# This is hardcoded to CPython 2.7 for win64, which is good
# enough for what we need currently. If we need psutil for Python 3
# in the future that could be added here as well.
self.virtualenv_manager._run_pip([
'download', '--dest', tmp, '--no-deps',
'--only-binary', ':all:', '--platform', 'win_amd64',
'--implementation', 'cp', '--python-version', '27',
'--abi', 'none', '--disable-pip-version-check',
packages[0]
])
self._extract(tmp, vendor_dir)
shutil.copyfile(tmpspec.name, spec)
self.repository.add_remove_files(vendor_dir)
def _get_test_metadata(self):
all_tests = NamedTemporaryFile(mode='wb')
pickle.dump(ALL_TESTS, all_tests)
all_tests.flush()
self._temp_files.append(all_tests)
test_defaults = NamedTemporaryFile(mode='wb')
pickle.dump(TEST_DEFAULTS, test_defaults)
test_defaults.flush()
self._temp_files.append(test_defaults)
rv = TestMetadata(all_tests.name, "/firefox/", test_defaults=test_defaults.name)
rv._wpt_loaded = True # Don't try to load the wpt manifest
return rv
def _get_test_metadata(self):
f = NamedTemporaryFile()
f.write(ALL_TESTS_JSON)
f.flush()
self._temp_files.append(f)
return TestMetadata(filename=f.name)
class TestLineEndings(unittest.TestCase):
"""
Unit tests for the Context class
"""
def setUp(self):
self.pp = Preprocessor()
self.pp.out = StringIO()
self.f = NamedTemporaryFile(mode='wb')
def tearDown(self):
self.f.close()
def createFile(self, lineendings):
for line, ending in zip([b'a', b'#literal b', b'c'], lineendings):
self.f.write(line + ending)
self.f.flush()
def testMac(self):
self.createFile([b'\x0D'] * 3)
self.pp.do_include(self.f.name)
self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
def testUnix(self):
self.createFile([b'\x0A'] * 3)
self.pp.do_include(self.f.name)
self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
def testWindows(self):
self.createFile([b'\x0D\x0A'] * 3)
self.pp.do_include(self.f.name)
self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
def test_python_change_regenerate_everything(self):
"""If a Python file changes, we should attempt to rebuild everything."""
# We don't want to mutate files in the source directory because we want
# to be able to build from a read-only filesystem. So, we install a
# dummy module and rewrite the metadata to say it comes from the source
# directory.
#
# Hacking imp to accept a MockedFile doesn't appear possible. So for
# the first iteration we read from a temp file. The second iteration
# doesn't need to import, so we are fine with a mocked file.
fake_path = mozpath.join(OUR_DIR, 'fakemodule.py')
with NamedTemporaryFile('wt') as fh:
fh.write('# Original content')
fh.flush()
mod = imp.load_source('mozwebidlcodegen.fakemodule', fh.name)
mod.__file__ = fake_path
args = self._get_manager_args()
m1 = WebIDLCodegenManager(**args)
with MockedOpen({fake_path: '# Original content'}):
old_exists = os.path.exists
try:
def exists(p):
if p == fake_path:
return True
return old_exists(p)
os.path.exists = exists
result = m1.generate_build_files()
l = len(result.inputs)
with open(fake_path, 'wt') as fh:
fh.write('# Modified content')
m2 = WebIDLCodegenManager(**args)
result = m2.generate_build_files()
self.assertEqual(len(result.inputs), l)
result = m2.generate_build_files()
self.assertEqual(len(result.inputs), 0)
finally:
os.path.exists = old_exists
del sys.modules['mozwebidlcodegen.fakemodule']
def _get_test_metadata(self):
all_tests = NamedTemporaryFile(mode='wb')
pickle.dump(ALL_TESTS, all_tests)
all_tests.flush()
self._temp_files.append(all_tests)
test_defaults = NamedTemporaryFile(mode='wb')
pickle.dump(TEST_DEFAULTS, test_defaults)
test_defaults.flush()
self._temp_files.append(test_defaults)
return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
def createSSLServerCertificate(build, srcDir):
certutil = build.get_binary_path(what="certutil")
pk12util = build.get_binary_path(what="pk12util")
with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
pgoCAPath = os.path.join(srcDir, "pgoca.p12")
pwfile.write("\n")
if not dbFilesExist(srcDir):
# Make sure all DB files from src are really deleted
unlinkDbFiles(srcDir)
# Create certification database for ssltunnel
status = runUtil(certutil, ["-N", "-d", srcDir, "-f", pwfile.name])
if status:
return status
status = runUtil(pk12util, [
"-i", pgoCAPath, "-w", pwfile.name, "-d", srcDir, "-k",
pwfile.name
])
if status:
return status
# Generate automatic certificate
locations = ServerLocations(
os.path.join(build.topsrcdir, "build", "pgo",
"server-locations.txt"))
iterator = iter(locations)
# Skips the first entry, I don't know why: bug 879740
iterator.next()
locationsParam = ""
firstLocation = ""
for loc in iterator:
if loc.scheme == "https" and "nocert" not in loc.options:
customCertOption = False
customCertRE = re.compile("^cert=(?:\w+)")
for option in loc.options:
match = customCertRE.match(option)
if match:
customCertOption = True
break
if not customCertOption:
if len(locationsParam) > 0:
locationsParam += ","
locationsParam += loc.host
if firstLocation == "":
firstLocation = loc.host
if not firstLocation:
print "Nothing to generate, no automatic secure hosts specified"
else:
createRandomFile(rndfile)
runUtil(certutil, [
"-D", "-n", "pgo server certificate", "-d", srcDir, "-z",
rndfile.name, "-f", pwfile.name
])
# Ignore the result, the certificate may not be present when new database is being built
status = runUtil(certutil, [
"-S", "-s",
"CN=%s" % firstLocation, "-t", "Pu,,", "-c",
"pgo temporary ca", "-m", "2", "-8", locationsParam, "-v",
"120", "-n", "pgo server certificate", "-d", srcDir, "-z",
rndfile.name, "-f", pwfile.name
])
if status:
return status
status = runUtil(certutil, [
"-S", "-s", "CN=Imminently Distrusted End Entity", "-t", "P,,",
"-c", "pgo temporary ca", "-k", "rsa", "-g", "2048", "-Z",
"SHA256", "-m", "1519140221", "-n", "imminently_distrusted",
"-v", "120", "-8", "imminently-distrusted.example.com", "-d",
srcDir, "-z", rndfile.name, "-f", pwfile.name
])
if status:
return status
"""
As of February 2018, there are 15 more certificates which are not created by
this script. See bug 1441338:
selfsigned Pu,u,u
Unknown CA Cu,u,u
escapeattack1 Pu,u,u
untrustedandexpired Pu,u,u
alternateTrustedAuthority Cu,u,u
dynamicPinningGood Pu,u,u
staticPinningBad Pu,u,u
sha1_end_entity Pu,u,u
bug413909cert u,u,u
untrusted Pu,u,u
escapeattack2 Pu,u,u
expired Pu,u,u
dynamicPinningBad Pu,u,u
sha256_end_entity Pu,u,u
"""
return 0
def constructCertDatabase(build, srcDir):
certutil = build.get_binary_path(what="certutil")
pk12util = build.get_binary_path(what="pk12util")
openssl = distutils.spawn.find_executable("openssl")
pycert = os.path.join(build.topsrcdir, "security", "manager", "ssl",
"tests", "unit", "pycert.py")
pykey = os.path.join(build.topsrcdir, "security", "manager", "ssl",
"tests", "unit", "pykey.py")
with NamedTemporaryFile() as pwfile, TemporaryDirectory() as pemfolder:
pwfile.write("\n")
pwfile.flush()
if dbFilesExist(srcDir):
# Make sure all DB files from src are really deleted
unlinkDbFiles(srcDir)
# Copy all .certspec and .keyspec files to a temporary directory
for root, dirs, files in os.walk(srcDir):
for spec in [
i for i in files
if i.endswith(".certspec") or i.endswith(".keyspec")
]:
shutil.copyfile(os.path.join(root, spec),
os.path.join(pemfolder, spec))
# Write a certspec for the "server-locations.txt" file to that temporary directory
pgoserver_certspec = os.path.join(pemfolder, "pgoserver.certspec")
if os.path.exists(pgoserver_certspec):
raise Exception("{} already exists, which isn't allowed".format(
pgoserver_certspec))
with open(pgoserver_certspec, "w") as fd:
writeCertspecForServerLocations(fd)
# Generate certs for all certspecs
for root, dirs, files in os.walk(pemfolder):
for certspec in [i for i in files if i.endswith(".certspec")]:
name = certspec.split(".certspec")[0]
pem = os.path.join(pemfolder, "{}.cert.pem".format(name))
print("Generating public certificate {} (pem={})".format(
name, pem))
with open(os.path.join(root, certspec), "r") as certspec_file:
certspec_data = certspec_file.read()
with open(pem, "w") as pem_file:
status = runUtil(pycert, [],
inputdata=certspec_data,
outputstream=pem_file)
if status:
return status
status = runUtil(certutil, [
"-A", "-n", name, "-t", "P,,", "-i", pem, "-d", srcDir,
"-f", pwfile.name
])
if status:
return status
for keyspec in [i for i in files if i.endswith(".keyspec")]:
parts = keyspec.split(".")
name = parts[0]
key_type = parts[1]
if key_type not in ["ca", "client", "server"]:
raise Exception(
"{}: keyspec filenames must be of the form XXX.client.keyspec "
"or XXX.ca.keyspec (key_type={})".format(
keyspec, key_type))
key_pem = os.path.join(pemfolder, "{}.key.pem".format(name))
print("Generating private key {} (pem={})".format(
name, key_pem))
with open(os.path.join(root, keyspec), "r") as keyspec_file:
keyspec_data = keyspec_file.read()
with open(key_pem, "w") as pem_file:
status = runUtil(pykey, [],
inputdata=keyspec_data,
outputstream=pem_file)
if status:
return status
cert_pem = os.path.join(pemfolder, "{}.cert.pem".format(name))
if not os.path.exists(cert_pem):
raise Exception(
"There has to be a corresponding certificate named {} for "
"the keyspec {}".format(cert_pem, keyspec))
p12 = os.path.join(pemfolder, "{}.key.p12".format(name))
print("Converting private key {} to PKCS12 (p12={})".format(
key_pem, p12))
status = runUtil(openssl, [
"pkcs12", "-export", "-inkey", key_pem, "-in", cert_pem,
"-name", name, "-out", p12, "-passout",
"file:" + pwfile.name
])
if status:
return status
print("Importing private key {} to database".format(key_pem))
status = runUtil(pk12util, [
"-i", p12, "-d", srcDir, "-w", pwfile.name, "-k",
pwfile.name
])
if status:
return status
if key_type == "ca":
shutil.copyfile(cert_pem,
os.path.join(srcDir, "{}.ca".format(name)))
elif key_type == "client":
shutil.copyfile(
p12, os.path.join(srcDir, "{}.client".format(name)))
elif key_type == "server":
pass # Nothing to do for server keys
else:
raise Exception(
"State error: Unknown keyspec key_type: {}".format(
key_type))
return 0
def setUp(self):
self.pp = Preprocessor()
self.pp.out = StringIO()
self.f = NamedTemporaryFile(mode='wb')
def createSSLServerCertificate(build, srcDir):
certutil = build.get_binary_path(what="certutil")
pk12util = build.get_binary_path(what="pk12util")
with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile:
pgoCAPath = os.path.join(srcDir, "pgoca.p12")
pwfile.write("\n")
if not dbFilesExist(srcDir):
# Make sure all DB files from src are really deleted
unlinkDbFiles(srcDir)
# Create certification database for ssltunnel
status = runUtil(certutil, ["-N", "-d", srcDir, "-f", pwfile.name])
if status:
return status
status = runUtil(pk12util, [
"-i", pgoCAPath, "-w", pwfile.name, "-d", srcDir, "-k",
pwfile.name
])
if status:
return status
# Generate automatic certificate
locations = ServerLocations(
os.path.join(build.topsrcdir, "build", "pgo",
"server-locations.txt"))
iterator = iter(locations)
# Skips the first entry, I don't know why: bug 879740
iterator.next()
locationsParam = ""
firstLocation = ""
for loc in iterator:
if loc.scheme == "https" and "nocert" not in loc.options:
customCertOption = False
customCertRE = re.compile("^cert=(?:\w+)")
for option in loc.options:
match = customCertRE.match(option)
if match:
customCertOption = True
break
if not customCertOption:
if len(locationsParam) > 0:
locationsParam += ","
locationsParam += loc.host
if firstLocation == "":
firstLocation = loc.host
if not firstLocation:
print "Nothing to generate, no automatic secure hosts specified"
else:
createRandomFile(rndfile)
runUtil(certutil, [
"-D", "-n", "pgo server certificate", "-d", srcDir, "-z",
rndfile.name, "-f", pwfile.name
])
# Ignore the result, the certificate may not be present when new database is being built
status = runUtil(certutil, [
"-S", "-s",
"CN=%s" % firstLocation, "-t", "Pu,,", "-c",
"pgo temporary ca", "-m", "2", "-8", locationsParam, "-v",
"120", "-n", "pgo server certificate", "-d", srcDir, "-z",
rndfile.name, "-f", pwfile.name
])
if status:
return status
return 0