def _get_test_metadata(self): f = NamedTemporaryFile() f.write(ALL_TESTS_JSON) f.flush() self._temp_files.append(f) return TestMetadata(filename=f.name)
def createCertificateAuthority(build, srcDir): certutil = build.get_binary_path(what="certutil") pk12util = build.get_binary_path(what="pk12util") #TODO: mozfile.TemporaryDirectory tempDbDir = tempfile.mkdtemp() with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile: pgoCAModulePathSrc = os.path.join(srcDir, "pgoca.p12") pgoCAPathSrc = os.path.join(srcDir, "pgoca.ca") pwfile.write("\n") # Create temporary certification database for CA generation status = runUtil(certutil, ["-N", "-d", tempDbDir, "-f", pwfile.name]) if status: return status createRandomFile(rndfile) status = runUtil(certutil, [ "-S", "-d", tempDbDir, "-s", "CN=Temporary Certificate Authority, O=Mozilla Testing, OU=Profile Guided Optimization", "-t", "C,,", "-x", "-m", "1", "-v", "120", "-n", "pgo temporary ca", "-2", "-f", pwfile.name, "-z", rndfile.name ], "Y\n0\nN\n") if status: return status status = runUtil(certutil, [ "-L", "-d", tempDbDir, "-n", "pgo temporary ca", "-a", "-o", pgoCAPathSrc, "-f", pwfile.name ]) if status: return status status = runUtil(pk12util, [ "-o", pgoCAModulePathSrc, "-n", "pgo temporary ca", "-d", tempDbDir, "-w", pwfile.name, "-k", pwfile.name ]) if status: return status shutil.rmtree(tempDbDir) return 0
def _get_test_metadata(self): all_tests = NamedTemporaryFile(mode='wb') pickle.dump(ALL_TESTS, all_tests) all_tests.flush() self._temp_files.append(all_tests) test_defaults = NamedTemporaryFile(mode='wb') pickle.dump(TEST_DEFAULTS, test_defaults) test_defaults.flush() self._temp_files.append(test_defaults) return TestMetadata(all_tests.name, test_defaults=test_defaults.name)
def vendor(self, packages=None, with_windows_wheel=False): self.populate_logger() self.log_manager.enable_unstructured() vendor_dir = mozpath.join(self.topsrcdir, os.path.join('third_party', 'python')) packages = packages or [] if with_windows_wheel and len(packages) != 1: raise Exception( '--with-windows-wheel is only supported for a single package!') self._activate_virtualenv() pip_compile = os.path.join(self.virtualenv_manager.bin_path, 'pip-compile') if not os.path.exists(pip_compile): path = os.path.normpath( os.path.join(self.topsrcdir, 'third_party', 'python', 'pip-tools')) self.virtualenv_manager.install_pip_package(path, vendored=True) spec = os.path.join(vendor_dir, 'requirements.in') requirements = os.path.join(vendor_dir, 'requirements.txt') with NamedTemporaryFile('w') as tmpspec: shutil.copyfile(spec, tmpspec.name) self._update_packages(tmpspec.name, packages) # resolve the dependencies and update requirements.txt subprocess.check_output([ pip_compile, tmpspec.name, '--no-header', '--no-index', '--output-file', requirements, '--generate-hashes' ]) with TemporaryDirectory() as tmp: # use requirements.txt to download archived source distributions of all packages self.virtualenv_manager._run_pip([ 'download', '-r', requirements, '--no-deps', '--dest', tmp, '--no-binary', ':all:', '--disable-pip-version-check' ]) if with_windows_wheel: # This is hardcoded to CPython 2.7 for win64, which is good # enough for what we need currently. If we need psutil for Python 3 # in the future that could be added here as well. self.virtualenv_manager._run_pip([ 'download', '--dest', tmp, '--no-deps', '--only-binary', ':all:', '--platform', 'win_amd64', '--implementation', 'cp', '--python-version', '27', '--abi', 'none', '--disable-pip-version-check', packages[0] ]) self._extract(tmp, vendor_dir) shutil.copyfile(tmpspec.name, spec) self.repository.add_remove_files(vendor_dir)
def _get_test_metadata(self): all_tests = NamedTemporaryFile(mode='wb') pickle.dump(ALL_TESTS, all_tests) all_tests.flush() self._temp_files.append(all_tests) test_defaults = NamedTemporaryFile(mode='wb') pickle.dump(TEST_DEFAULTS, test_defaults) test_defaults.flush() self._temp_files.append(test_defaults) rv = TestMetadata(all_tests.name, "/firefox/", test_defaults=test_defaults.name) rv._wpt_loaded = True # Don't try to load the wpt manifest return rv
class TestLineEndings(unittest.TestCase): """ Unit tests for the Context class """ def setUp(self): self.pp = Preprocessor() self.pp.out = StringIO() self.f = NamedTemporaryFile(mode='wb') def tearDown(self): self.f.close() def createFile(self, lineendings): for line, ending in zip([b'a', b'#literal b', b'c'], lineendings): self.f.write(line + ending) self.f.flush() def testMac(self): self.createFile([b'\x0D'] * 3) self.pp.do_include(self.f.name) self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') def testUnix(self): self.createFile([b'\x0A'] * 3) self.pp.do_include(self.f.name) self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n') def testWindows(self): self.createFile([b'\x0D\x0A'] * 3) self.pp.do_include(self.f.name) self.assertEquals(self.pp.out.getvalue(), 'a\nb\nc\n')
def test_python_change_regenerate_everything(self): """If a Python file changes, we should attempt to rebuild everything.""" # We don't want to mutate files in the source directory because we want # to be able to build from a read-only filesystem. So, we install a # dummy module and rewrite the metadata to say it comes from the source # directory. # # Hacking imp to accept a MockedFile doesn't appear possible. So for # the first iteration we read from a temp file. The second iteration # doesn't need to import, so we are fine with a mocked file. fake_path = mozpath.join(OUR_DIR, 'fakemodule.py') with NamedTemporaryFile('wt') as fh: fh.write('# Original content') fh.flush() mod = imp.load_source('mozwebidlcodegen.fakemodule', fh.name) mod.__file__ = fake_path args = self._get_manager_args() m1 = WebIDLCodegenManager(**args) with MockedOpen({fake_path: '# Original content'}): old_exists = os.path.exists try: def exists(p): if p == fake_path: return True return old_exists(p) os.path.exists = exists result = m1.generate_build_files() l = len(result.inputs) with open(fake_path, 'wt') as fh: fh.write('# Modified content') m2 = WebIDLCodegenManager(**args) result = m2.generate_build_files() self.assertEqual(len(result.inputs), l) result = m2.generate_build_files() self.assertEqual(len(result.inputs), 0) finally: os.path.exists = old_exists del sys.modules['mozwebidlcodegen.fakemodule']
def createSSLServerCertificate(build, srcDir): certutil = build.get_binary_path(what="certutil") pk12util = build.get_binary_path(what="pk12util") with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile: pgoCAPath = os.path.join(srcDir, "pgoca.p12") pwfile.write("\n") if not dbFilesExist(srcDir): # Make sure all DB files from src are really deleted unlinkDbFiles(srcDir) # Create certification database for ssltunnel status = runUtil(certutil, ["-N", "-d", srcDir, "-f", pwfile.name]) if status: return status status = runUtil(pk12util, [ "-i", pgoCAPath, "-w", pwfile.name, "-d", srcDir, "-k", pwfile.name ]) if status: return status # Generate automatic certificate locations = ServerLocations( os.path.join(build.topsrcdir, "build", "pgo", "server-locations.txt")) iterator = iter(locations) # Skips the first entry, I don't know why: bug 879740 iterator.next() locationsParam = "" firstLocation = "" for loc in iterator: if loc.scheme == "https" and "nocert" not in loc.options: customCertOption = False customCertRE = re.compile("^cert=(?:\w+)") for option in loc.options: match = customCertRE.match(option) if match: customCertOption = True break if not customCertOption: if len(locationsParam) > 0: locationsParam += "," locationsParam += loc.host if firstLocation == "": firstLocation = loc.host if not firstLocation: print "Nothing to generate, no automatic secure hosts specified" else: createRandomFile(rndfile) runUtil(certutil, [ "-D", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name ]) # Ignore the result, the certificate may not be present when new database is being built status = runUtil(certutil, [ "-S", "-s", "CN=%s" % firstLocation, "-t", "Pu,,", "-c", "pgo temporary ca", "-m", "2", "-8", locationsParam, "-v", "120", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name ]) if status: return status status = runUtil(certutil, [ "-S", "-s", "CN=Imminently Distrusted End Entity", "-t", "P,,", "-c", "pgo temporary ca", "-k", "rsa", "-g", "2048", "-Z", "SHA256", "-m", "1519140221", "-n", "imminently_distrusted", "-v", "120", "-8", "imminently-distrusted.example.com", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name ]) if status: return status """ As of February 2018, there are 15 more certificates which are not created by this script. See bug 1441338: selfsigned Pu,u,u Unknown CA Cu,u,u escapeattack1 Pu,u,u untrustedandexpired Pu,u,u alternateTrustedAuthority Cu,u,u dynamicPinningGood Pu,u,u staticPinningBad Pu,u,u sha1_end_entity Pu,u,u bug413909cert u,u,u untrusted Pu,u,u escapeattack2 Pu,u,u expired Pu,u,u dynamicPinningBad Pu,u,u sha256_end_entity Pu,u,u """ return 0
def constructCertDatabase(build, srcDir): certutil = build.get_binary_path(what="certutil") pk12util = build.get_binary_path(what="pk12util") openssl = distutils.spawn.find_executable("openssl") pycert = os.path.join(build.topsrcdir, "security", "manager", "ssl", "tests", "unit", "pycert.py") pykey = os.path.join(build.topsrcdir, "security", "manager", "ssl", "tests", "unit", "pykey.py") with NamedTemporaryFile() as pwfile, TemporaryDirectory() as pemfolder: pwfile.write("\n") pwfile.flush() if dbFilesExist(srcDir): # Make sure all DB files from src are really deleted unlinkDbFiles(srcDir) # Copy all .certspec and .keyspec files to a temporary directory for root, dirs, files in os.walk(srcDir): for spec in [ i for i in files if i.endswith(".certspec") or i.endswith(".keyspec") ]: shutil.copyfile(os.path.join(root, spec), os.path.join(pemfolder, spec)) # Write a certspec for the "server-locations.txt" file to that temporary directory pgoserver_certspec = os.path.join(pemfolder, "pgoserver.certspec") if os.path.exists(pgoserver_certspec): raise Exception("{} already exists, which isn't allowed".format( pgoserver_certspec)) with open(pgoserver_certspec, "w") as fd: writeCertspecForServerLocations(fd) # Generate certs for all certspecs for root, dirs, files in os.walk(pemfolder): for certspec in [i for i in files if i.endswith(".certspec")]: name = certspec.split(".certspec")[0] pem = os.path.join(pemfolder, "{}.cert.pem".format(name)) print("Generating public certificate {} (pem={})".format( name, pem)) with open(os.path.join(root, certspec), "r") as certspec_file: certspec_data = certspec_file.read() with open(pem, "w") as pem_file: status = runUtil(pycert, [], inputdata=certspec_data, outputstream=pem_file) if status: return status status = runUtil(certutil, [ "-A", "-n", name, "-t", "P,,", "-i", pem, "-d", srcDir, "-f", pwfile.name ]) if status: return status for keyspec in [i for i in files if i.endswith(".keyspec")]: parts = keyspec.split(".") name = parts[0] key_type = parts[1] if key_type not in ["ca", "client", "server"]: raise Exception( "{}: keyspec filenames must be of the form XXX.client.keyspec " "or XXX.ca.keyspec (key_type={})".format( keyspec, key_type)) key_pem = os.path.join(pemfolder, "{}.key.pem".format(name)) print("Generating private key {} (pem={})".format( name, key_pem)) with open(os.path.join(root, keyspec), "r") as keyspec_file: keyspec_data = keyspec_file.read() with open(key_pem, "w") as pem_file: status = runUtil(pykey, [], inputdata=keyspec_data, outputstream=pem_file) if status: return status cert_pem = os.path.join(pemfolder, "{}.cert.pem".format(name)) if not os.path.exists(cert_pem): raise Exception( "There has to be a corresponding certificate named {} for " "the keyspec {}".format(cert_pem, keyspec)) p12 = os.path.join(pemfolder, "{}.key.p12".format(name)) print("Converting private key {} to PKCS12 (p12={})".format( key_pem, p12)) status = runUtil(openssl, [ "pkcs12", "-export", "-inkey", key_pem, "-in", cert_pem, "-name", name, "-out", p12, "-passout", "file:" + pwfile.name ]) if status: return status print("Importing private key {} to database".format(key_pem)) status = runUtil(pk12util, [ "-i", p12, "-d", srcDir, "-w", pwfile.name, "-k", pwfile.name ]) if status: return status if key_type == "ca": shutil.copyfile(cert_pem, os.path.join(srcDir, "{}.ca".format(name))) elif key_type == "client": shutil.copyfile( p12, os.path.join(srcDir, "{}.client".format(name))) elif key_type == "server": pass # Nothing to do for server keys else: raise Exception( "State error: Unknown keyspec key_type: {}".format( key_type)) return 0
def setUp(self): self.pp = Preprocessor() self.pp.out = StringIO() self.f = NamedTemporaryFile(mode='wb')
def createSSLServerCertificate(build, srcDir): certutil = build.get_binary_path(what="certutil") pk12util = build.get_binary_path(what="pk12util") with NamedTemporaryFile() as pwfile, NamedTemporaryFile() as rndfile: pgoCAPath = os.path.join(srcDir, "pgoca.p12") pwfile.write("\n") if not dbFilesExist(srcDir): # Make sure all DB files from src are really deleted unlinkDbFiles(srcDir) # Create certification database for ssltunnel status = runUtil(certutil, ["-N", "-d", srcDir, "-f", pwfile.name]) if status: return status status = runUtil(pk12util, [ "-i", pgoCAPath, "-w", pwfile.name, "-d", srcDir, "-k", pwfile.name ]) if status: return status # Generate automatic certificate locations = ServerLocations( os.path.join(build.topsrcdir, "build", "pgo", "server-locations.txt")) iterator = iter(locations) # Skips the first entry, I don't know why: bug 879740 iterator.next() locationsParam = "" firstLocation = "" for loc in iterator: if loc.scheme == "https" and "nocert" not in loc.options: customCertOption = False customCertRE = re.compile("^cert=(?:\w+)") for option in loc.options: match = customCertRE.match(option) if match: customCertOption = True break if not customCertOption: if len(locationsParam) > 0: locationsParam += "," locationsParam += loc.host if firstLocation == "": firstLocation = loc.host if not firstLocation: print "Nothing to generate, no automatic secure hosts specified" else: createRandomFile(rndfile) runUtil(certutil, [ "-D", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name ]) # Ignore the result, the certificate may not be present when new database is being built status = runUtil(certutil, [ "-S", "-s", "CN=%s" % firstLocation, "-t", "Pu,,", "-c", "pgo temporary ca", "-m", "2", "-8", locationsParam, "-v", "120", "-n", "pgo server certificate", "-d", srcDir, "-z", rndfile.name, "-f", pwfile.name ]) if status: return status return 0