Exemplo n.º 1
0
class B2GHazardBuild(PurgeMixin, B2GBuildBaseScript):
    all_actions = [
        'clobber',
        'checkout-tools',
        'checkout-sources',
        'get-blobs',
        'update-source-manifest',
        'clobber-shell',
        'configure-shell',
        'build-shell',
        'clobber-analysis',
        'setup-analysis',
        'run-analysis',
        'collect-analysis-output',
        'upload-analysis',
        'check-expectations',
    ]

    default_actions = [
        'clobber',
        'checkout-tools',
        'checkout-sources',
        'get-blobs',
        'clobber-shell',
        'configure-shell',
        'build-shell',
        'clobber-analysis',
        'setup-analysis',
        'run-analysis',
        'collect-analysis-output',
        'upload-analysis',
        'check-expectations',
    ]

    def __init__(self):
        super(B2GHazardBuild, self).__init__(
            config_options=[],
            config={
                'default_vcs': 'hgtool',
                'ccache': False,
                'mozilla_dir': 'build/gecko',

                'upload_ssh_server': None,
                'upload_remote_basepath': None,
                'enable_try_uploads': True,
            },
            all_actions=B2GHazardBuild.all_actions,
            default_actions=B2GHazardBuild.default_actions,
        )

        self.buildid = None
        self.analysis = HazardAnalysis()

    def _pre_config_lock(self, rw_config):
        super(B2GHazardBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config:
            self.config['is_automation'] = True
        else:
            self.config['is_automation'] = False

        dirs = self.query_abs_dirs()
        replacements = self.config['env_replacements'].copy()
        for k, v in replacements.items():
            replacements[k] = v % dirs

        self.env = self.query_env(replace_dict=replacements,
                                  partial_env=self.config['partial_env'],
                                  purge_env=nuisance_env_vars)

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(B2GHazardBuild, self).query_abs_dirs()

        abs_work_dir = abs_dirs['abs_work_dir']
        dirs = {
            'b2g_src':
                abs_work_dir,
            'target_compiler_base':
                os.path.join(abs_dirs['abs_work_dir'], 'target_compiler'),
            'shell_objdir':
                os.path.join(abs_work_dir, self.config['shell-objdir']),
            'mozharness_scriptdir':
                os.path.abspath(os.path.dirname(__file__)),
            'abs_analysis_dir':
                os.path.join(abs_work_dir, self.config['analysis-dir']),
            'abs_analyzed_objdir':
                os.path.join(abs_work_dir, self.config['srcdir'], self.config['analysis-objdir']),
            'analysis_scriptdir':
                os.path.join(abs_dirs['gecko_src'], self.config['analysis-scriptdir'])
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        elif 'branch' in self.config:
            # Used for locally testing try vs non-try
            return self.config['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs['analysis_scriptdir'], self.config['compiler_manifest'])

    def query_b2g_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs['analysis_scriptdir'], self.config['b2g_compiler_manifest'])

    def query_sixgill_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs['analysis_scriptdir'], self.config['sixgill_manifest'])

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        if self.buildbot_config and 'properties' in self.buildbot_config:
            self.buildid = self.buildbot_config['properties'].get('buildid')
        if not self.buildid:
            self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
        return self.buildid

    def query_upload_ssh_server(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_server']
        else:
            return self.config['upload_ssh_server']

    def query_upload_ssh_key(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            key = self.buildbot_config['properties']['upload_ssh_key']
        else:
            key = self.config['upload_ssh_key']
        if self.mock_enabled and not key.startswith("/"):
            key = "/home/mock_mozilla/.ssh/" + key
        return key

    def query_upload_ssh_user(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_user']
        else:
            return self.config['upload_ssh_user']

    def query_product(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['product']
        else:
            return self.config['product']

    def query_upload_remote_basepath(self):
        if self.config.get('upload_remote_basepath'):
            return self.config['upload_remote_basepath']
        else:
            return "/pub/mozilla.org/{product}".format(
                product=self.query_product(),
            )

    def query_upload_remote_baseuri(self):
        baseuri = self.config.get('upload_remote_baseuri')
        if self.buildbot_config and 'properties' in self.buildbot_config:
            buildprops = self.buildbot_config['properties']
            if 'upload_remote_baseuri' in buildprops:
                baseuri = buildprops['upload_remote_baseuri']
        return baseuri.strip("/") if baseuri else None

    def query_upload_label(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['platform']
        else:
            return self.config.get('upload_label')

    def query_upload_path(self):
        branch = self.query_branch()

        common = {
            'basepath': self.query_upload_remote_basepath(),
            'branch': branch,
            'target': self.query_upload_label(),
        }

        if branch == 'try':
            if not self.config['enable_try_uploads']:
                return None
            try:
                user = self.buildbot_config['sourcestamp']['changes'][0]['who']
            except (KeyError, TypeError):
                user = "******"
            return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
                user=user,
                rev=self.query_revision(),
                **common
            )
        else:
            return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
                buildid=self.query_buildid(),
                **common
            )

    def query_do_upload(self):
        if self.query_branch() == 'try':
            return self.config.get('enable_try_uploads')
        return True

    def make_source_dir(self):
        dirs = self.query_abs_dirs()
        dest = dirs['b2g_src']
        if not os.path.exists(dest):
            self.mkdir_p(dest)

    # The gecko config is required before enabling mock, because it determines
    # what mock target to use.
    def enable_mock(self):
        self.load_gecko_config()
        super(B2GHazardBuild, self).enable_mock()

    # Actions {{{2
    def clobber(self):
        dirs = self.query_abs_dirs()
        PurgeMixin.clobber(
            self,
            always_clobber_dirs=[
                dirs['abs_upload_dir'],
            ],
        )

    def checkout_sources(self):
        self.make_source_dir()
        super(B2GHazardBuild, self).checkout_sources()

    def get_blobs(self):
        dirs = self.query_abs_dirs()
        self.tooltool_fetch(self.query_compiler_manifest(),
                            output_dir=dirs['abs_work_dir'])
        self.tooltool_fetch(self.query_sixgill_manifest(),
                            output_dir=dirs['abs_work_dir'])
        if not os.path.exists(dirs['target_compiler_base']):
            self.mkdir_p(dirs['target_compiler_base'])
        self.tooltool_fetch(self.query_b2g_compiler_manifest(),
                            output_dir=dirs['target_compiler_base'])

    def clobber_shell(self):
        dirs = self.query_abs_dirs()
        self.rmtree(dirs['shell_objdir'])

    def configure_shell(self):
        self.enable_mock()
        dirs = self.query_abs_dirs()

        if not os.path.exists(dirs['shell_objdir']):
            self.mkdir_p(dirs['shell_objdir'])

        js_src_dir = os.path.join(dirs['gecko_src'], 'js', 'src')
        rc = self.run_command(['autoconf-2.13'],
                              cwd=js_src_dir,
                              env=self.env,
                              error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("autoconf failed, can't continue.", exit_code=FAILURE)

        rc = self.run_command([os.path.join(js_src_dir, 'configure'),
                               '--enable-optimize',
                               '--disable-debug',
                               '--enable-ctypes',
                               '--with-system-nspr',
                               '--without-intl-api'],
                              cwd=dirs['shell_objdir'],
                              env=self.env,
                              error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("Configure failed, can't continue.", exit_code=FAILURE)

        self.disable_mock()

    def build_shell(self):
        self.enable_mock()
        dirs = self.query_abs_dirs()

        rc = self.run_command(['make', '-j', str(self.config.get('concurrency', 4)), '-s'],
                              cwd=dirs['shell_objdir'],
                              env=self.env,
                              error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("Build failed, can't continue.", exit_code=FAILURE)

        self.disable_mock()

    def clobber_analysis(self):
        dirs = self.query_abs_dirs()
        self.rmtree(dirs['abs_analysis_dir'])
        self.rmtree(dirs['abs_analyzed_objdir'])

    def setup_analysis(self):
        dirs = self.query_abs_dirs()
        analysis_dir = dirs['abs_analysis_dir']

        if not os.path.exists(analysis_dir):
            self.mkdir_p(analysis_dir)

        values = {
            'js': os.path.join(dirs['shell_objdir'], 'dist', 'bin', 'js'),
            'analysis_scriptdir': dirs['analysis_scriptdir'],
            'source_objdir': dirs['abs_analyzed_objdir'],
            'source': os.path.join(dirs['work_dir'], 'source'),
            'sixgill': os.path.join(dirs['work_dir'], self.config['sixgill']),
            'sixgill_bin': os.path.join(dirs['work_dir'], self.config['sixgill_bin']),
        }
        defaults = """
js = '%(js)s'
analysis_scriptdir = '%(analysis_scriptdir)s'
objdir = '%(source_objdir)s'
source = '%(source)s'
sixgill = '%(sixgill)s'
sixgill_bin = '%(sixgill_bin)s'
jobs = 2
""" % values

        #defaults_path = os.path.join(analysis_dir, 'defaults.py')
        defaults_path = os.path.join(analysis_dir, 'defaults.py')
        file(defaults_path, "w").write(defaults)
        self.log("Wrote analysis config file " + defaults_path)

        build_command = self.config['build_command']
        self.copyfile(os.path.join(dirs['mozharness_scriptdir'],
                                   os.path.join('spidermonkey', build_command)),
                      os.path.join(analysis_dir, build_command),
                      copystat=True)

    def run_analysis(self):
        dirs = self.query_abs_dirs()

        env = self.query_build_env().copy()
        self.enable_mock()

        gonk_misc = os.path.join(dirs['b2g_src'], 'gonk-misc')
        mozconfig = os.path.join(gonk_misc, 'hazard-analysis-config')
        mozconfig_text = ''
        mozconfig_text += 'CXXFLAGS="-Wno-attributes"\n'
        mozconfig_text += '. "%s/default-gecko-config"\n' % gonk_misc
        basecc = os.path.join(dirs['abs_work_dir'], self.config['sixgill'], 'scripts', 'wrap_gcc', 'basecc')
        mozconfig_text += "ac_add_options --with-compiler-wrapper=" + basecc + "\n"
        mozconfig_text += "ac_add_options --without-ccache\n"
        file(mozconfig, "w").write(mozconfig_text)

        # Stuff I set in my .userconfig for manual builds
        env['B2G_SOURCE'] = dirs['b2g_src']
        env['MOZCONFIG_PATH'] = mozconfig
        env['GECKO_PATH'] = dirs['gecko_src']
        env['TARGET_TOOLS_PREFIX'] = os.path.join(dirs['abs_work_dir'], self.config['b2g_target_compiler_prefix'])

        try:
            self.analysis.run(self, env=env, error_list=B2GMakefileErrorList)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def collect_analysis_output(self):
        self.analysis.collect_output(self)

    def upload_analysis(self):
        if not self.query_do_upload():
            self.info("Uploads disabled for this build. Skipping...")
            return

        self.enable_mock()

        try:
            self.analysis.upload_results(self)
        except HazardError as e:
            self.error(e)
            self.return_code = WARNINGS

        self.disable_mock()

    def check_expectations(self):
        try:
            self.analysis.check_expectations(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)
Exemplo n.º 2
0
class SpidermonkeyBuild(MockMixin,
                        PurgeMixin, BaseScript,
                        VCSMixin, BuildbotMixin, TooltoolMixin, TransferMixin):
    config_options = [
        [["--repo"], {
            "dest": "repo",
            "help": "which gecko repo to get spidermonkey from",
        }],
        [["--source"], {
            "dest": "source",
            "help": "directory containing gecko source tree (instead of --repo)",
        }],
        [["--revision"], {
            "dest": "revision",
        }],
        [["--branch"], {
            "dest": "branch",
        }],
        [["--vcs-share-base"], {
            "dest": "vcs_share_base",
            "help": "base directory for shared repositories",
        }],
        [["-j"], {
            "dest": "concurrency",
            "type": int,
            "default": 4,
            "help": "number of simultaneous jobs used while building the shell " +
                    "(currently ignored for the analyzed build",
        }],
    ]

    def __init__(self):
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            # other stuff
                            all_actions=[
                                'purge',
                                'checkout-tools',

                                # First, build an optimized JS shell for running the analysis
                                'checkout-source',
                                'get-blobs',
                                'clobber-shell',
                                'configure-shell',
                                'build-shell',

                                # Next, build a tree with the analysis plugin
                                # active. Note that we are using the same
                                # checkout for the JS shell build and the build
                                # of the source to be analyzed, which is a
                                # little unnecessary (no need to rebuild the JS
                                # shell all the time). (Different objdir,
                                # though.)
                                'clobber-analysis',
                                'setup-analysis',
                                'run-analysis',
                                'collect-analysis-output',
                                'upload-analysis',
                                'check-expectations',
                            ],
                            default_actions=[
                                'purge',
                                'checkout-tools',
                                'checkout-source',
                                'get-blobs',
                                'clobber-shell',
                                'configure-shell',
                                'build-shell',
                                'clobber-analysis',
                                'setup-analysis',
                                'run-analysis',
                                'collect-analysis-output',
                                'upload-analysis',
                                'check-expectations',
                            ],
                            config={
                                'default_vcs': 'hgtool',
                                'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
                                'ccache': True,
                                'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
                                'tools_repo': 'https://hg.mozilla.org/build/tools',

                                'upload_ssh_server': None,
                                'upload_remote_basepath': None,
                                'enable_try_uploads': True,
                                'source': None,
                            },
        )

        self.buildid = None
        self.analysis = HazardAnalysis()

    def _pre_config_lock(self, rw_config):
        if self.config['source']:
            self.config['srcdir'] = self.config['source']
        super(SpidermonkeyBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config is None:
            self.info("Reading buildbot build properties...")
            self.read_buildbot_config()

        if self.buildbot_config:
            bb_props = [('mock_target', 'mock_target', None),
                        ('base_bundle_urls', 'hgtool_base_bundle_urls', None),
                        ('base_mirror_urls', 'hgtool_base_mirror_urls', None),
                        ('hgurl', 'hgurl', None),
                        ('clobberer_url', 'clobberer_url', 'https://api.pub.build.mozilla.org/clobberer/lastclobber'),
                        ('purge_minsize', 'purge_minsize', 15),
                        ('purge_maxage', 'purge_maxage', None),
                        ('purge_skip', 'purge_skip', None),
                        ('force_clobber', 'force_clobber', None),
                        ]
            buildbot_props = self.buildbot_config.get('properties', {})
            for bb_prop, cfg_prop, default in bb_props:
                if not self.config.get(cfg_prop) and buildbot_props.get(bb_prop, default):
                    self.config[cfg_prop] = buildbot_props.get(bb_prop, default)
            self.config['is_automation'] = True
        else:
            self.config['is_automation'] = False

        dirs = self.query_abs_dirs()
        replacements = self.config['env_replacements'].copy()
        for k,v in replacements.items():
            replacements[k] = v % dirs

        self.env = self.query_env(replace_dict=replacements,
                                  partial_env=self.config['partial_env'],
                                  purge_env=nuisance_env_vars)

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = BaseScript.query_abs_dirs(self)

        abs_work_dir = abs_dirs['abs_work_dir']
        dirs = {
            'shell_objdir':
                os.path.join(abs_work_dir, self.config['shell-objdir']),
            'mozharness_scriptdir':
                os.path.abspath(os.path.dirname(__file__)),
            'abs_analysis_dir':
                os.path.join(abs_work_dir, self.config['analysis-dir']),
            'abs_analyzed_objdir':
                os.path.join(abs_work_dir, self.config['srcdir'], self.config['analysis-objdir']),
            'analysis_scriptdir':
                os.path.join(self.config['srcdir'], self.config['analysis-scriptdir']),
            'abs_tools_dir':
                os.path.join(abs_dirs['base_work_dir'], 'tools'),
            'gecko_src':
                os.path.join(abs_work_dir, self.config['srcdir']),
            'abs_blob_upload_dir':
                os.path.join(abs_work_dir, 'blobber_upload_dir'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs

        return self.abs_dirs

    def query_repo(self):
        if self.config.get('repo'):
            return self.config['repo']
        elif self.buildbot_config and 'properties' in self.buildbot_config:
            return self.config['hgurl'] + self.buildbot_config['properties']['repo_path']
        else:
            return None

    def query_revision(self):
        if 'revision' in self.buildbot_properties:
            revision = self.buildbot_properties['revision']
        elif self.buildbot_config and 'sourcestamp' in self.buildbot_config:
            revision = self.buildbot_config['sourcestamp']['revision']
        else:
            # Useful for local testing. In actual use, this would always be
            # None.
            revision = self.config.get('revision')

        return revision

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        elif 'branch' in self.config:
            # Used for locally testing try vs non-try
            return self.config['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['compiler_manifest'])
        if os.path.exists(manifest):
            return manifest
        return os.path.join(dirs['abs_work_dir'], self.config['compiler_manifest'])

    def query_sixgill_manifest(self):
        dirs = self.query_abs_dirs()
        manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['sixgill_manifest'])
        if os.path.exists(manifest):
            return manifest
        return os.path.join(dirs['abs_work_dir'], self.config['sixgill_manifest'])

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        if self.buildbot_config and 'properties' in self.buildbot_config:
            self.buildid = self.buildbot_config['properties'].get('buildid')
        if not self.buildid:
            self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
        return self.buildid

    def query_upload_ssh_server(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_server']
        else:
            return self.config['upload_ssh_server']

    def query_upload_ssh_key(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            key = self.buildbot_config['properties']['upload_ssh_key']
        else:
            key = self.config['upload_ssh_key']
        if self.mock_enabled and not key.startswith("/"):
            key = "/home/mock_mozilla/.ssh/" + key
        return key

    def query_upload_ssh_user(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_user']
        else:
            return self.config['upload_ssh_user']

    def query_product(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['product']
        else:
            return self.config['product']

    def query_upload_remote_basepath(self):
        if self.config.get('upload_remote_basepath'):
            return self.config['upload_remote_basepath']
        else:
            return "/pub/mozilla.org/{product}".format(
                product=self.query_product(),
            )

    def query_upload_remote_baseuri(self):
        baseuri = self.config.get('upload_remote_baseuri')
        if self.buildbot_config and 'properties' in self.buildbot_config:
            buildprops = self.buildbot_config['properties']
            if 'upload_remote_baseuri' in buildprops:
                baseuri = buildprops['upload_remote_baseuri']
        return baseuri.strip("/") if baseuri else None

    def query_target(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['platform']
        else:
            return self.config.get('target')

    def query_upload_path(self):
        branch = self.query_branch()

        common = {
            'basepath': self.query_upload_remote_basepath(),
            'branch': branch,
            'target': self.query_target(),
        }

        if branch == 'try':
            if not self.config['enable_try_uploads']:
                return None
            try:
                user = self.buildbot_config['sourcestamp']['changes'][0]['who']
            except (KeyError, TypeError):
                user = "******"
            return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
                user=user,
                rev=self.query_revision(),
                **common
            )
        else:
            return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
                buildid=self.query_buildid(),
                **common
            )

    def query_do_upload(self):
        if self.query_branch() == 'try':
            return self.config.get('enable_try_uploads')
        return True

    # Actions {{{2
    def purge(self):
        dirs = self.query_abs_dirs()
        self.info("purging, abs_upload_dir=" + dirs['abs_upload_dir'])
        PurgeMixin.clobber(
            self,
            always_clobber_dirs=[
                dirs['abs_upload_dir'],
            ],
        )

    def checkout_tools(self):
        dirs = self.query_abs_dirs()
        rev = self.vcs_checkout(
            vcs='hg',  # Don't have hgtool.py yet
            repo=self.config['tools_repo'],
            clean=False,
            dest=dirs['abs_tools_dir'],
        )
        self.set_buildbot_property("tools_revision", rev, write_to_file=True)

    def do_checkout_source(self):
        # --source option means to use an existing source directory instead of checking one out.
        if self.config['source']:
            return

        dirs = self.query_abs_dirs()
        dest = dirs['gecko_src']

        # Pre-create the directory to appease the share extension
        if not os.path.exists(dest):
            self.mkdir_p(dest)

        rev = self.vcs_checkout(
            repo=self.query_repo(),
            dest=dest,
            revision=self.query_revision(),
            branch=self.config.get('branch'),
            clean=True,
        )
        self.set_buildbot_property('source_revision', rev, write_to_file=True)

    def checkout_source(self):
        try:
            self.do_checkout_source()
        except Exception as e:
            self.fatal("checkout failed: " + str(e), exit_code=RETRY)

    def get_blobs(self):
        work_dir = self.query_abs_dirs()['abs_work_dir']
        if not os.path.exists(work_dir):
            self.mkdir_p(work_dir)
        self.tooltool_fetch(self.query_compiler_manifest(), output_dir=work_dir)
        self.tooltool_fetch(self.query_sixgill_manifest(), output_dir=work_dir)

    def clobber_shell(self):
        self.analysis.clobber_shell(self)

    def configure_shell(self):
        self.enable_mock()

        try:
            self.analysis.configure_shell(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def build_shell(self):
        self.enable_mock()

        try:
            self.analysis.build_shell(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def clobber_analysis(self):
        self.analysis.clobber(self)

    def setup_analysis(self):
        self.analysis.setup(self)

    def run_analysis(self):
        self.enable_mock()

        upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
        if not os.path.exists(upload_dir):
            self.mkdir_p(upload_dir)

        env = self.env.copy()
        env['MOZ_UPLOAD_DIR'] = upload_dir

        try:
            self.analysis.run(self, env=env, error_list=MakefileErrorList)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def collect_analysis_output(self):
        self.analysis.collect_output(self)

    def upload_analysis(self):
        if not self.config['is_automation']:
            return

        if not self.query_do_upload():
            self.info("Uploads disabled for this build. Skipping...")
            return

        self.enable_mock()

        try:
            self.analysis.upload_results(self)
        except HazardError as e:
            self.error(e)
            self.return_code = WARNINGS

        self.disable_mock()

    def check_expectations(self):
        try:
            self.analysis.check_expectations(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)
Exemplo n.º 3
0
class B2GHazardBuild(PurgeMixin, B2GBuildBaseScript):
    all_actions = [
        "clobber",
        "checkout-tools",
        "checkout-sources",
        "get-blobs",
        "update-source-manifest",
        "clobber-shell",
        "configure-shell",
        "build-shell",
        "clobber-analysis",
        "setup-analysis",
        "run-analysis",
        "collect-analysis-output",
        "upload-analysis",
        "check-expectations",
    ]

    default_actions = [
        "clobber",
        "checkout-tools",
        "checkout-sources",
        "get-blobs",
        "clobber-shell",
        "configure-shell",
        "build-shell",
        "clobber-analysis",
        "setup-analysis",
        "run-analysis",
        "collect-analysis-output",
        "upload-analysis",
        "check-expectations",
    ]

    def __init__(self):
        super(B2GHazardBuild, self).__init__(
            config_options=[],
            config={
                "default_vcs": "hgtool",
                "ccache": False,
                "mozilla_dir": "build/goanna",
                "upload_ssh_server": None,
                "upload_remote_basepath": None,
                "enable_try_uploads": True,
            },
            all_actions=B2GHazardBuild.all_actions,
            default_actions=B2GHazardBuild.default_actions,
        )

        self.buildid = None
        self.analysis = HazardAnalysis()

    def _pre_config_lock(self, rw_config):
        super(B2GHazardBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config:
            self.config["is_automation"] = True
        else:
            self.config["is_automation"] = False

        dirs = self.query_abs_dirs()
        replacements = self.config["env_replacements"].copy()
        for k, v in replacements.items():
            replacements[k] = v % dirs

        self.env = self.query_env(
            replace_dict=replacements, partial_env=self.config["partial_env"], purge_env=nuisance_env_vars
        )

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(B2GHazardBuild, self).query_abs_dirs()

        abs_work_dir = abs_dirs["abs_work_dir"]
        dirs = {
            "b2g_src": abs_work_dir,
            "target_compiler_base": os.path.join(abs_dirs["abs_work_dir"], "target_compiler"),
            "shell_objdir": os.path.join(abs_work_dir, self.config["shell-objdir"]),
            "mozharness_scriptdir": os.path.abspath(os.path.dirname(__file__)),
            "abs_analysis_dir": os.path.join(abs_work_dir, self.config["analysis-dir"]),
            "abs_analyzed_objdir": os.path.join(abs_work_dir, self.config["srcdir"], self.config["analysis-objdir"]),
            "analysis_scriptdir": os.path.join(abs_dirs["goanna_src"], self.config["analysis-scriptdir"]),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_branch(self):
        if self.buildbot_config and "properties" in self.buildbot_config:
            return self.buildbot_config["properties"]["branch"]
        elif "branch" in self.config:
            # Used for locally testing try vs non-try
            return self.config["branch"]
        else:
            return os.path.basename(self.query_repo())

    def query_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs["analysis_scriptdir"], self.config["compiler_manifest"])

    def query_b2g_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs["analysis_scriptdir"], self.config["b2g_compiler_manifest"])

    def query_sixgill_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs["analysis_scriptdir"], self.config["sixgill_manifest"])

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        if self.buildbot_config and "properties" in self.buildbot_config:
            self.buildid = self.buildbot_config["properties"].get("buildid")
        if not self.buildid:
            self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
        return self.buildid

    def query_upload_ssh_server(self):
        if self.buildbot_config and "properties" in self.buildbot_config:
            return self.buildbot_config["properties"]["upload_ssh_server"]
        else:
            return self.config["upload_ssh_server"]

    def query_upload_ssh_key(self):
        if self.buildbot_config and "properties" in self.buildbot_config:
            key = self.buildbot_config["properties"]["upload_ssh_key"]
        else:
            key = self.config["upload_ssh_key"]
        if self.mock_enabled and not key.startswith("/"):
            key = "/home/mock_mozilla/.ssh/" + key
        return key

    def query_upload_ssh_user(self):
        if self.buildbot_config and "properties" in self.buildbot_config:
            return self.buildbot_config["properties"]["upload_ssh_user"]
        else:
            return self.config["upload_ssh_user"]

    def query_product(self):
        if self.buildbot_config and "properties" in self.buildbot_config:
            return self.buildbot_config["properties"]["product"]
        else:
            return self.config["product"]

    def query_upload_remote_basepath(self):
        if self.config.get("upload_remote_basepath"):
            return self.config["upload_remote_basepath"]
        else:
            return "/pub/mozilla.org/{product}".format(product=self.query_product())

    def query_upload_remote_baseuri(self):
        baseuri = self.config.get("upload_remote_baseuri")
        if self.buildbot_config and "properties" in self.buildbot_config:
            buildprops = self.buildbot_config["properties"]
            if "upload_remote_baseuri" in buildprops:
                baseuri = buildprops["upload_remote_baseuri"]
        return baseuri.strip("/") if baseuri else None

    def query_upload_label(self):
        if self.buildbot_config and "properties" in self.buildbot_config:
            return self.buildbot_config["properties"]["platform"]
        else:
            return self.config.get("upload_label")

    def query_upload_path(self):
        branch = self.query_branch()

        common = {
            "basepath": self.query_upload_remote_basepath(),
            "branch": branch,
            "target": self.query_upload_label(),
        }

        if branch == "try":
            if not self.config["enable_try_uploads"]:
                return None
            try:
                user = self.buildbot_config["sourcestamp"]["changes"][0]["who"]
            except (KeyError, TypeError):
                user = "******"
            return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
                user=user, rev=self.query_revision(), **common
            )
        else:
            return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
                buildid=self.query_buildid(), **common
            )

    def query_do_upload(self):
        if self.query_branch() == "try":
            return self.config.get("enable_try_uploads")
        return True

    def make_source_dir(self):
        dirs = self.query_abs_dirs()
        dest = dirs["b2g_src"]
        if not os.path.exists(dest):
            self.mkdir_p(dest)

    # The goanna config is required before enabling mock, because it determines
    # what mock target to use.
    def enable_mock(self):
        self.load_goanna_config()
        super(B2GHazardBuild, self).enable_mock()

    # Actions {{{2
    def clobber(self):
        dirs = self.query_abs_dirs()
        PurgeMixin.clobber(self, always_clobber_dirs=[dirs["abs_upload_dir"]])

    def checkout_sources(self):
        self.make_source_dir()
        super(B2GHazardBuild, self).checkout_sources()

    def get_blobs(self):
        dirs = self.query_abs_dirs()
        self.tooltool_fetch(self.query_compiler_manifest(), "sh " + self.config["compiler_setup"], dirs["abs_work_dir"])
        self.tooltool_fetch(self.query_sixgill_manifest(), "sh " + self.config["sixgill_setup"], dirs["abs_work_dir"])
        if not os.path.exists(dirs["target_compiler_base"]):
            self.mkdir_p(dirs["target_compiler_base"])
        self.tooltool_fetch(
            self.query_b2g_compiler_manifest(), "sh " + self.config["compiler_setup"], dirs["target_compiler_base"]
        )

    def clobber_shell(self):
        dirs = self.query_abs_dirs()
        self.rmtree(dirs["shell_objdir"])

    def configure_shell(self):
        self.enable_mock()
        dirs = self.query_abs_dirs()

        if not os.path.exists(dirs["shell_objdir"]):
            self.mkdir_p(dirs["shell_objdir"])

        js_src_dir = os.path.join(dirs["goanna_src"], "js", "src")
        rc = self.run_command(["autoconf-2.13"], cwd=js_src_dir, env=self.env, error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("autoconf failed, can't continue.", exit_code=FAILURE)

        rc = self.run_command(
            [
                os.path.join(js_src_dir, "configure"),
                "--enable-optimize",
                "--disable-debug",
                "--enable-ctypes",
                "--with-system-nspr",
                "--without-intl-api",
            ],
            cwd=dirs["shell_objdir"],
            env=self.env,
            error_list=MakefileErrorList,
        )
        if rc != 0:
            self.fatal("Configure failed, can't continue.", exit_code=FAILURE)

        self.disable_mock()

    def build_shell(self):
        self.enable_mock()
        dirs = self.query_abs_dirs()

        rc = self.run_command(
            ["make", "-j", str(self.config.get("concurrency", 4)), "-s"],
            cwd=dirs["shell_objdir"],
            env=self.env,
            error_list=MakefileErrorList,
        )
        if rc != 0:
            self.fatal("Build failed, can't continue.", exit_code=FAILURE)

        self.disable_mock()

    def clobber_analysis(self):
        dirs = self.query_abs_dirs()
        self.rmtree(dirs["abs_analysis_dir"])
        self.rmtree(dirs["abs_analyzed_objdir"])

    def setup_analysis(self):
        dirs = self.query_abs_dirs()
        analysis_dir = dirs["abs_analysis_dir"]

        if not os.path.exists(analysis_dir):
            self.mkdir_p(analysis_dir)

        values = {
            "js": os.path.join(dirs["shell_objdir"], "dist", "bin", "js"),
            "analysis_scriptdir": dirs["analysis_scriptdir"],
            "source_objdir": dirs["abs_analyzed_objdir"],
            "source": os.path.join(dirs["work_dir"], "source"),
            "sixgill": os.path.join(dirs["work_dir"], self.config["sixgill"]),
            "sixgill_bin": os.path.join(dirs["work_dir"], self.config["sixgill_bin"]),
        }
        defaults = (
            """
js = '%(js)s'
analysis_scriptdir = '%(analysis_scriptdir)s'
objdir = '%(source_objdir)s'
source = '%(source)s'
sixgill = '%(sixgill)s'
sixgill_bin = '%(sixgill_bin)s'
jobs = 2
"""
            % values
        )

        # defaults_path = os.path.join(analysis_dir, 'defaults.py')
        defaults_path = os.path.join(analysis_dir, "defaults.py")
        file(defaults_path, "w").write(defaults)
        self.log("Wrote analysis config file " + defaults_path)

        build_command = self.config["build_command"]
        self.copyfile(
            os.path.join(dirs["mozharness_scriptdir"], os.path.join("spidermonkey", build_command)),
            os.path.join(analysis_dir, build_command),
            copystat=True,
        )

    def run_analysis(self):
        dirs = self.query_abs_dirs()

        env = self.query_build_env().copy()
        self.enable_mock()

        gonk_misc = os.path.join(dirs["b2g_src"], "gonk-misc")
        mozconfig = os.path.join(gonk_misc, "hazard-analysis-config")
        mozconfig_text = ""
        mozconfig_text += 'CXXFLAGS="-Wno-attributes"\n'
        mozconfig_text += '. "%s/default-goanna-config"\n' % gonk_misc
        basecc = os.path.join(dirs["abs_work_dir"], self.config["sixgill"], "scripts", "wrap_gcc", "basecc")
        mozconfig_text += "ac_add_options --with-compiler-wrapper=" + basecc + "\n"
        mozconfig_text += "ac_add_options --without-ccache\n"
        file(mozconfig, "w").write(mozconfig_text)

        # Stuff I set in my .userconfig for manual builds
        env["B2G_SOURCE"] = dirs["b2g_src"]
        env["MOZCONFIG_PATH"] = mozconfig
        env["GECKO_PATH"] = dirs["goanna_src"]
        env["TARGET_TOOLS_PREFIX"] = os.path.join(dirs["abs_work_dir"], self.config["b2g_target_compiler_prefix"])

        try:
            self.analysis.run(self, env=env, error_list=B2GMakefileErrorList)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def collect_analysis_output(self):
        self.analysis.collect_output(self)

    def upload_analysis(self):
        if not self.query_do_upload():
            self.info("Uploads disabled for this build. Skipping...")
            return

        self.enable_mock()

        try:
            self.analysis.upload_results(self)
        except HazardError as e:
            self.error(e)
            self.return_code = WARNINGS

        self.disable_mock()

    def check_expectations(self):
        try:
            self.analysis.check_expectations(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)
Exemplo n.º 4
0
class SpidermonkeyBuild(MockMixin,
                        PurgeMixin, BaseScript,
                        VCSMixin, BuildbotMixin, TooltoolMixin, TransferMixin):
    config_options = [
        [["--repo"], {
            "dest": "repo",
            "help": "which gecko repo to get spidermonkey from",
        }],
        [["--source"], {
            "dest": "source",
            "help": "directory containing gecko source tree (instead of --repo)",
        }],
        [["--revision"], {
            "dest": "revision",
        }],
        [["--branch"], {
            "dest": "branch",
        }],
        [["--vcs-share-base"], {
            "dest": "vcs_share_base",
            "help": "base directory for shared repositories",
        }],
        [["-j"], {
            "dest": "concurrency",
            "type": int,
            "default": 4,
            "help": "number of simultaneous jobs used while building the shell " +
                    "(currently ignored for the analyzed build",
        }],
    ]

    def __init__(self):
        BaseScript.__init__(self,
                            config_options=self.config_options,
                            # other stuff
                            all_actions=[
                                'purge',
                                'checkout-tools',

                                # First, build an optimized JS shell for running the analysis
                                'checkout-source',
                                'get-blobs',
                                'clobber-shell',
                                'configure-shell',
                                'build-shell',

                                # Next, build a tree with the analysis plugin
                                # active. Note that we are using the same
                                # checkout for the JS shell build and the build
                                # of the source to be analyzed, which is a
                                # little unnecessary (no need to rebuild the JS
                                # shell all the time). (Different objdir,
                                # though.)
                                'clobber-analysis',
                                'setup-analysis',
                                'run-analysis',
                                'collect-analysis-output',
                                'upload-analysis',
                                'check-expectations',
                            ],
                            default_actions=[
                                'purge',
                                'checkout-tools',
                                'checkout-source',
                                'get-blobs',
                                'clobber-shell',
                                'configure-shell',
                                'build-shell',
                                'clobber-analysis',
                                'setup-analysis',
                                'run-analysis',
                                'collect-analysis-output',
                                # Temporary - see bug 1211402
                                #'upload-analysis',
                                'check-expectations',
                            ],
                            config={
                                'default_vcs': 'hgtool',
                                'vcs_share_base': os.environ.get('HG_SHARE_BASE_DIR'),
                                'ccache': True,
                                'buildbot_json_path': os.environ.get('PROPERTIES_FILE'),
                                'tools_repo': 'https://hg.mozilla.org/build/tools',

                                'upload_ssh_server': None,
                                'upload_remote_basepath': None,
                                'enable_try_uploads': True,
                                'source': None,
                                'stage_product': 'firefox',
                            },
        )

        self.buildid = None
        self.analysis = HazardAnalysis()

    def _pre_config_lock(self, rw_config):
        if self.config['source']:
            self.config['srcdir'] = self.config['source']
        super(SpidermonkeyBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config is None:
            self.info("Reading buildbot build properties...")
            self.read_buildbot_config()

        if self.buildbot_config:
            bb_props = [('mock_target', 'mock_target', None),
                        ('base_bundle_urls', 'hgtool_base_bundle_urls', None),
                        ('base_mirror_urls', 'hgtool_base_mirror_urls', None),
                        ('hgurl', 'hgurl', None),
                        ('clobberer_url', 'clobberer_url', 'https://api.pub.build.mozilla.org/clobberer/lastclobber'),
                        ('purge_minsize', 'purge_minsize', 15),
                        ('purge_maxage', 'purge_maxage', None),
                        ('purge_skip', 'purge_skip', None),
                        ('force_clobber', 'force_clobber', None),
                        ]
            buildbot_props = self.buildbot_config.get('properties', {})
            for bb_prop, cfg_prop, default in bb_props:
                if not self.config.get(cfg_prop) and buildbot_props.get(bb_prop, default):
                    self.config[cfg_prop] = buildbot_props.get(bb_prop, default)
            self.config['is_automation'] = True
        else:
            self.config['is_automation'] = False

        dirs = self.query_abs_dirs()
        replacements = self.config['env_replacements'].copy()
        for k,v in replacements.items():
            replacements[k] = v % dirs

        self.env = self.query_env(replace_dict=replacements,
                                  partial_env=self.config['partial_env'],
                                  purge_env=nuisance_env_vars)

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = BaseScript.query_abs_dirs(self)

        abs_work_dir = abs_dirs['abs_work_dir']
        dirs = {
            'shell_objdir':
                os.path.join(abs_work_dir, self.config['shell-objdir']),
            'mozharness_scriptdir':
                os.path.abspath(os.path.dirname(__file__)),
            'abs_analysis_dir':
                os.path.join(abs_work_dir, self.config['analysis-dir']),
            'abs_analyzed_objdir':
                os.path.join(abs_work_dir, self.config['srcdir'], self.config['analysis-objdir']),
            'analysis_scriptdir':
                os.path.join(self.config['srcdir'], self.config['analysis-scriptdir']),
            'abs_tools_dir':
                os.path.join(abs_dirs['base_work_dir'], 'tools'),
            'gecko_src':
                os.path.join(abs_work_dir, self.config['srcdir']),
            'abs_blob_upload_dir':
                os.path.join(abs_work_dir, 'blobber_upload_dir'),
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs

        return self.abs_dirs

    def query_repo(self):
        if self.config.get('repo'):
            return self.config['repo']
        elif self.buildbot_config and 'properties' in self.buildbot_config:
            return self.config['hgurl'] + self.buildbot_config['properties']['repo_path']
        else:
            return None

    def query_revision(self):
        if 'revision' in self.buildbot_properties:
            revision = self.buildbot_properties['revision']
        elif self.buildbot_config and 'sourcestamp' in self.buildbot_config:
            revision = self.buildbot_config['sourcestamp']['revision']
        else:
            # Useful for local testing. In actual use, this would always be
            # None.
            revision = self.config.get('revision')

        return revision

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        elif 'branch' in self.config:
            # Used for locally testing try vs non-try
            return self.config['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['compiler_manifest'])
        if os.path.exists(manifest):
            return manifest
        return os.path.join(dirs['abs_work_dir'], self.config['compiler_manifest'])

    def query_sixgill_manifest(self):
        dirs = self.query_abs_dirs()
        manifest = os.path.join(dirs['abs_work_dir'], dirs['analysis_scriptdir'], self.config['sixgill_manifest'])
        if os.path.exists(manifest):
            return manifest
        return os.path.join(dirs['abs_work_dir'], self.config['sixgill_manifest'])

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        if self.buildbot_config and 'properties' in self.buildbot_config:
            self.buildid = self.buildbot_config['properties'].get('buildid')
        if not self.buildid:
            self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
        return self.buildid

    def query_upload_ssh_server(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_server']
        else:
            return self.config['upload_ssh_server']

    def query_upload_ssh_key(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            key = self.buildbot_config['properties']['upload_ssh_key']
        else:
            key = self.config['upload_ssh_key']
        if self.mock_enabled and not key.startswith("/"):
            key = "/home/mock_mozilla/.ssh/" + key
        return key

    def query_upload_ssh_user(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_user']
        else:
            return self.config['upload_ssh_user']

    def query_product(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['product']
        else:
            return self.config['product']

    def query_upload_remote_basepath(self):
        if self.config.get('upload_remote_basepath'):
            return self.config['upload_remote_basepath']
        else:
            return "/pub/mozilla.org/{product}".format(
                product=self.query_product(),
            )

    def query_upload_remote_baseuri(self):
        baseuri = self.config.get('upload_remote_baseuri')
        if self.buildbot_config and 'properties' in self.buildbot_config:
            buildprops = self.buildbot_config['properties']
            if 'upload_remote_baseuri' in buildprops:
                baseuri = buildprops['upload_remote_baseuri']
        return baseuri.strip("/") if baseuri else None

    def query_target(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['platform']
        else:
            return self.config.get('target')

    def query_upload_path(self):
        branch = self.query_branch()

        common = {
            'basepath': self.query_upload_remote_basepath(),
            'branch': branch,
            'target': self.query_target(),
        }

        if branch == 'try':
            if not self.config['enable_try_uploads']:
                return None
            try:
                user = self.buildbot_config['sourcestamp']['changes'][0]['who']
            except (KeyError, TypeError):
                user = "******"
            return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
                user=user,
                rev=self.query_revision(),
                **common
            )
        else:
            return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
                buildid=self.query_buildid(),
                **common
            )

    def query_do_upload(self):
        if self.query_branch() == 'try':
            return self.config.get('enable_try_uploads')
        return True

    # Actions {{{2
    def purge(self):
        dirs = self.query_abs_dirs()
        self.info("purging, abs_upload_dir=" + dirs['abs_upload_dir'])
        PurgeMixin.clobber(
            self,
            always_clobber_dirs=[
                dirs['abs_upload_dir'],
            ],
        )

    def checkout_tools(self):
        dirs = self.query_abs_dirs()

        # If running from within a directory also passed as the --source dir,
        # this has the danger of clobbering <source>/tools/
        if self.config['source']:
            srcdir = self.config['source']
            if os.path.samefile(srcdir, os.path.dirname(dirs['abs_tools_dir'])):
                raise Exception("Cannot run from source checkout to avoid overwriting subdirs")

        rev = self.vcs_checkout(
            vcs='hg',  # Don't have hgtool.py yet
            repo=self.config['tools_repo'],
            clean=False,
            dest=dirs['abs_tools_dir'],
        )
        self.set_buildbot_property("tools_revision", rev, write_to_file=True)

    def do_checkout_source(self):
        # --source option means to use an existing source directory instead of checking one out.
        if self.config['source']:
            return

        dirs = self.query_abs_dirs()
        dest = dirs['gecko_src']

        # Pre-create the directory to appease the share extension
        if not os.path.exists(dest):
            self.mkdir_p(dest)

        rev = self.vcs_checkout(
            repo=self.query_repo(),
            dest=dest,
            revision=self.query_revision(),
            branch=self.config.get('branch'),
            clean=True,
        )
        self.set_buildbot_property('source_revision', rev, write_to_file=True)

    def checkout_source(self):
        try:
            self.do_checkout_source()
        except Exception as e:
            self.fatal("checkout failed: " + str(e), exit_code=RETRY)

    def get_blobs(self):
        work_dir = self.query_abs_dirs()['abs_work_dir']
        if not os.path.exists(work_dir):
            self.mkdir_p(work_dir)
        self.tooltool_fetch(self.query_compiler_manifest(), output_dir=work_dir)
        self.tooltool_fetch(self.query_sixgill_manifest(), output_dir=work_dir)

    def clobber_shell(self):
        self.analysis.clobber_shell(self)

    def configure_shell(self):
        self.enable_mock()

        try:
            self.analysis.configure_shell(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def build_shell(self):
        self.enable_mock()

        try:
            self.analysis.build_shell(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def clobber_analysis(self):
        self.analysis.clobber(self)

    def setup_analysis(self):
        self.analysis.setup(self)

    def run_analysis(self):
        self.enable_mock()

        upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
        if not os.path.exists(upload_dir):
            self.mkdir_p(upload_dir)

        env = self.env.copy()
        env['MOZ_UPLOAD_DIR'] = upload_dir

        try:
            self.analysis.run(self, env=env, error_list=MakefileErrorList)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def collect_analysis_output(self):
        self.analysis.collect_output(self)

    def upload_analysis(self):
        if not self.config['is_automation']:
            return

        if not self.query_do_upload():
            self.info("Uploads disabled for this build. Skipping...")
            return

        self.enable_mock()

        try:
            self.analysis.upload_results(self)
        except HazardError as e:
            self.error(e)
            self.return_code = WARNINGS

        self.disable_mock()

    def check_expectations(self):
        try:
            self.analysis.check_expectations(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)
Exemplo n.º 5
0
class B2GHazardBuild(PurgeMixin, B2GBuildBaseScript):
    all_actions = [
        'clobber',
        'checkout-tools',
        'checkout-sources',
        'get-blobs',
        'update-source-manifest',
        'clobber-shell',
        'configure-shell',
        'build-shell',
        'clobber-analysis',
        'setup-analysis',
        'run-analysis',
        'collect-analysis-output',
        'upload-analysis',
        'check-expectations',
    ]

    default_actions = [
        'clobber',
        'checkout-tools',
        'checkout-sources',
        'get-blobs',
        'clobber-shell',
        'configure-shell',
        'build-shell',
        'clobber-analysis',
        'setup-analysis',
        'run-analysis',
        'collect-analysis-output',
        'upload-analysis',
        'check-expectations',
    ]

    def __init__(self):
        super(B2GHazardBuild, self).__init__(
            config_options=[],
            config={
                'default_vcs': 'hgtool',
                'ccache': False,
                'mozilla_dir': 'build/gecko',
                'upload_ssh_server': None,
                'upload_remote_basepath': None,
                'enable_try_uploads': True,
            },
            all_actions=B2GHazardBuild.all_actions,
            default_actions=B2GHazardBuild.default_actions,
        )

        self.buildid = None
        self.analysis = HazardAnalysis()

    def _pre_config_lock(self, rw_config):
        super(B2GHazardBuild, self)._pre_config_lock(rw_config)

        if self.buildbot_config:
            self.config['is_automation'] = True
        else:
            self.config['is_automation'] = False

        dirs = self.query_abs_dirs()
        replacements = self.config['env_replacements'].copy()
        for k, v in replacements.items():
            replacements[k] = v % dirs

        self.env = self.query_env(replace_dict=replacements,
                                  partial_env=self.config['partial_env'],
                                  purge_env=nuisance_env_vars)

    def query_abs_dirs(self):
        if self.abs_dirs:
            return self.abs_dirs
        abs_dirs = super(B2GHazardBuild, self).query_abs_dirs()

        abs_work_dir = abs_dirs['abs_work_dir']
        dirs = {
            'b2g_src':
            abs_work_dir,
            'target_compiler_base':
            os.path.join(abs_dirs['abs_work_dir'], 'target_compiler'),
            'shell_objdir':
            os.path.join(abs_work_dir, self.config['shell-objdir']),
            'mozharness_scriptdir':
            os.path.abspath(os.path.dirname(__file__)),
            'abs_analysis_dir':
            os.path.join(abs_work_dir, self.config['analysis-dir']),
            'abs_analyzed_objdir':
            os.path.join(abs_work_dir, self.config['srcdir'],
                         self.config['analysis-objdir']),
            'analysis_scriptdir':
            os.path.join(abs_dirs['gecko_src'],
                         self.config['analysis-scriptdir'])
        }

        abs_dirs.update(dirs)
        self.abs_dirs = abs_dirs
        return self.abs_dirs

    def query_branch(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['branch']
        elif 'branch' in self.config:
            # Used for locally testing try vs non-try
            return self.config['branch']
        else:
            return os.path.basename(self.query_repo())

    def query_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs['analysis_scriptdir'],
                            self.config['compiler_manifest'])

    def query_b2g_compiler_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs['analysis_scriptdir'],
                            self.config['b2g_compiler_manifest'])

    def query_sixgill_manifest(self):
        dirs = self.query_abs_dirs()
        return os.path.join(dirs['analysis_scriptdir'],
                            self.config['sixgill_manifest'])

    def query_buildid(self):
        if self.buildid:
            return self.buildid
        if self.buildbot_config and 'properties' in self.buildbot_config:
            self.buildid = self.buildbot_config['properties'].get('buildid')
        if not self.buildid:
            self.buildid = datetime.now().strftime("%Y%m%d%H%M%S")
        return self.buildid

    def query_upload_ssh_server(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_server']
        else:
            return self.config['upload_ssh_server']

    def query_upload_ssh_key(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            key = self.buildbot_config['properties']['upload_ssh_key']
        else:
            key = self.config['upload_ssh_key']
        if self.mock_enabled and not key.startswith("/"):
            key = "/home/mock_mozilla/.ssh/" + key
        return key

    def query_upload_ssh_user(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['upload_ssh_user']
        else:
            return self.config['upload_ssh_user']

    def query_product(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['product']
        else:
            return self.config['product']

    def query_upload_remote_basepath(self):
        if self.config.get('upload_remote_basepath'):
            return self.config['upload_remote_basepath']
        else:
            return "/pub/mozilla.org/{product}".format(
                product=self.query_product(), )

    def query_upload_remote_baseuri(self):
        baseuri = self.config.get('upload_remote_baseuri')
        if self.buildbot_config and 'properties' in self.buildbot_config:
            buildprops = self.buildbot_config['properties']
            if 'upload_remote_baseuri' in buildprops:
                baseuri = buildprops['upload_remote_baseuri']
        return baseuri.strip("/") if baseuri else None

    def query_upload_label(self):
        if self.buildbot_config and 'properties' in self.buildbot_config:
            return self.buildbot_config['properties']['platform']
        else:
            return self.config.get('upload_label')

    def query_upload_path(self):
        branch = self.query_branch()

        common = {
            'basepath': self.query_upload_remote_basepath(),
            'branch': branch,
            'target': self.query_upload_label(),
        }

        if branch == 'try':
            if not self.config['enable_try_uploads']:
                return None
            try:
                user = self.buildbot_config['sourcestamp']['changes'][0]['who']
            except (KeyError, TypeError):
                user = "******"
            return "{basepath}/try-builds/{user}-{rev}/{branch}-{target}".format(
                user=user, rev=self.query_revision(), **common)
        else:
            return "{basepath}/tinderbox-builds/{branch}-{target}/{buildid}".format(
                buildid=self.query_buildid(), **common)

    def query_do_upload(self):
        if self.query_branch() == 'try':
            return self.config.get('enable_try_uploads')
        return True

    def make_source_dir(self):
        dirs = self.query_abs_dirs()
        dest = dirs['b2g_src']
        if not os.path.exists(dest):
            self.mkdir_p(dest)

    # The gecko config is required before enabling mock, because it determines
    # what mock target to use.
    def enable_mock(self):
        self.load_gecko_config()
        super(B2GHazardBuild, self).enable_mock()

    # Actions {{{2
    def clobber(self):
        dirs = self.query_abs_dirs()
        PurgeMixin.clobber(
            self,
            always_clobber_dirs=[
                dirs['abs_upload_dir'],
            ],
        )

    def checkout_sources(self):
        self.make_source_dir()
        super(B2GHazardBuild, self).checkout_sources()

    def get_blobs(self):
        dirs = self.query_abs_dirs()
        self.tooltool_fetch(self.query_compiler_manifest(),
                            output_dir=dirs['abs_work_dir'])
        self.tooltool_fetch(self.query_sixgill_manifest(),
                            output_dir=dirs['abs_work_dir'])
        if not os.path.exists(dirs['target_compiler_base']):
            self.mkdir_p(dirs['target_compiler_base'])
        self.tooltool_fetch(self.query_b2g_compiler_manifest(),
                            output_dir=dirs['target_compiler_base'])

    def clobber_shell(self):
        dirs = self.query_abs_dirs()
        self.rmtree(dirs['shell_objdir'])

    def configure_shell(self):
        self.enable_mock()
        dirs = self.query_abs_dirs()

        if not os.path.exists(dirs['shell_objdir']):
            self.mkdir_p(dirs['shell_objdir'])

        js_src_dir = os.path.join(dirs['gecko_src'], 'js', 'src')
        rc = self.run_command(['autoconf-2.13'],
                              cwd=js_src_dir,
                              env=self.env,
                              error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("autoconf failed, can't continue.", exit_code=FAILURE)

        rc = self.run_command([
            os.path.join(js_src_dir,
                         'configure'), '--enable-optimize', '--disable-debug',
            '--enable-ctypes', '--with-system-nspr', '--without-intl-api'
        ],
                              cwd=dirs['shell_objdir'],
                              env=self.env,
                              error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("Configure failed, can't continue.", exit_code=FAILURE)

        self.disable_mock()

    def build_shell(self):
        self.enable_mock()
        dirs = self.query_abs_dirs()

        rc = self.run_command(
            ['make', '-j',
             str(self.config.get('concurrency', 4)), '-s'],
            cwd=dirs['shell_objdir'],
            env=self.env,
            error_list=MakefileErrorList)
        if rc != 0:
            self.fatal("Build failed, can't continue.", exit_code=FAILURE)

        self.disable_mock()

    def clobber_analysis(self):
        dirs = self.query_abs_dirs()
        self.rmtree(dirs['abs_analysis_dir'])
        self.rmtree(dirs['abs_analyzed_objdir'])

    def setup_analysis(self):
        dirs = self.query_abs_dirs()
        analysis_dir = dirs['abs_analysis_dir']

        if not os.path.exists(analysis_dir):
            self.mkdir_p(analysis_dir)

        values = {
            'js':
            os.path.join(dirs['shell_objdir'], 'dist', 'bin', 'js'),
            'analysis_scriptdir':
            dirs['analysis_scriptdir'],
            'source_objdir':
            dirs['abs_analyzed_objdir'],
            'source':
            os.path.join(dirs['work_dir'], 'source'),
            'sixgill':
            os.path.join(dirs['work_dir'], self.config['sixgill']),
            'sixgill_bin':
            os.path.join(dirs['work_dir'], self.config['sixgill_bin']),
        }
        defaults = """
js = '%(js)s'
analysis_scriptdir = '%(analysis_scriptdir)s'
objdir = '%(source_objdir)s'
source = '%(source)s'
sixgill = '%(sixgill)s'
sixgill_bin = '%(sixgill_bin)s'
jobs = 2
""" % values

        #defaults_path = os.path.join(analysis_dir, 'defaults.py')
        defaults_path = os.path.join(analysis_dir, 'defaults.py')
        file(defaults_path, "w").write(defaults)
        self.log("Wrote analysis config file " + defaults_path)

        build_command = self.config['build_command']
        self.copyfile(os.path.join(dirs['mozharness_scriptdir'],
                                   os.path.join('spidermonkey',
                                                build_command)),
                      os.path.join(analysis_dir, build_command),
                      copystat=True)

    def run_analysis(self):
        dirs = self.query_abs_dirs()

        env = self.query_build_env().copy()
        self.enable_mock()

        gonk_misc = os.path.join(dirs['b2g_src'], 'gonk-misc')
        mozconfig = os.path.join(gonk_misc, 'hazard-analysis-config')
        mozconfig_text = ''
        mozconfig_text += 'CXXFLAGS="-Wno-attributes"\n'
        mozconfig_text += '. "%s/default-gecko-config"\n' % gonk_misc
        basecc = os.path.join(dirs['abs_work_dir'], self.config['sixgill'],
                              'scripts', 'wrap_gcc', 'basecc')
        mozconfig_text += "ac_add_options --with-compiler-wrapper=" + basecc + "\n"
        mozconfig_text += "ac_add_options --without-ccache\n"
        file(mozconfig, "w").write(mozconfig_text)

        # Stuff I set in my .userconfig for manual builds
        env['B2G_SOURCE'] = dirs['b2g_src']
        env['MOZCONFIG_PATH'] = mozconfig
        env['GECKO_PATH'] = dirs['gecko_src']
        env['TARGET_TOOLS_PREFIX'] = os.path.join(
            dirs['abs_work_dir'], self.config['b2g_target_compiler_prefix'])

        try:
            self.analysis.run(self, env=env, error_list=B2GMakefileErrorList)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)

        self.disable_mock()

    def collect_analysis_output(self):
        self.analysis.collect_output(self)

    def upload_analysis(self):
        if not self.query_do_upload():
            self.info("Uploads disabled for this build. Skipping...")
            return

        self.enable_mock()

        try:
            self.analysis.upload_results(self)
        except HazardError as e:
            self.error(e)
            self.return_code = WARNINGS

        self.disable_mock()

    def check_expectations(self):
        try:
            self.analysis.check_expectations(self)
        except HazardError as e:
            self.fatal(e, exit_code=FAILURE)