Example #1
0
 def start(self):
     from scheduler import Tree
     loog = self.addLog('stdio')
     self.pending = 0
     properties = self.build.getProperties()
     self.rendered_tree = tree = properties.render(self.treename)
     l10nbuilds = properties.render(self.l10nbuilds)
     cp = ConfigParser()
     cp.read(l10nbuilds)
     repo = cp.get(tree, 'repo')
     branch = cp.get(tree, 'mozilla')
     path = cp.get(tree, 'l10n.ini')
     l10nbranch = cp.get(tree, 'l10n')
     locales = cp.get(tree, 'locales')
     if locales == 'all':
         alllocales = "yes"
     else:
         alllocales = "no"
         properties.update({'locales': filter(None, locales.split())},
                           "Build")
     self.tree = Tree(self.rendered_tree, repo, branch, l10nbranch, path)
     loog.addStdout('Loading l10n.inis for %s\n' % self.rendered_tree)
     logger.debug(
         'scheduler.l10n.tree', 'Loading l10n.inis for %s, alllocales: %s' %
         (self.rendered_tree, alllocales))
     self.loadIni(repo, branch, path, alllocales)
Example #2
0
    def start(self):
        from scheduler import Tree

        loog = self.addLog("stdio")
        self.pending = 0
        properties = self.build.getProperties()
        self.rendered_tree = tree = properties.render(self.treename)
        l10nbuilds = properties.render(self.l10nbuilds)
        cp = ConfigParser()
        cp.read(l10nbuilds)
        repo = cp.get(tree, "repo")
        branch = cp.get(tree, "mozilla")
        path = cp.get(tree, "l10n.ini")
        l10nbranch = cp.get(tree, "l10n")
        locales = cp.get(tree, "locales")
        if locales == "all":
            alllocales = "yes"
        else:
            alllocales = "no"
            properties.update({"locales": filter(None, locales.split())}, "Build")
        self.tree = Tree(self.rendered_tree, repo, branch, l10nbranch, path)
        loog.addStdout("Loading l10n.inis for %s\n" % self.rendered_tree)
        logger.debug(
            "scheduler.l10n.tree", "Loading l10n.inis for %s, alllocales: %s" % (self.rendered_tree, alllocales)
        )
        self.loadIni(repo, branch, path, alllocales)
Example #3
0
class TreeLoader(BuildStep):
    """BuildStep to load data from l10n.ini on remote repos.

    Does mostly just async network traffic, directly on the master,
    it wouldn't be more work there if we'd use the slave, and then
    marshall the data through the network back to the master.
    """

    def __init__(self, treename, l10nbuilds, cb=None, **kwargs):
        """Create a TreeLoader step. In addition to the standard arguments,
        treename is the name of the tree,
        l10nbuilds is the local ini file describing the builds,
        cb is a callback with signature (tree, changes=None)
        """
        BuildStep.__init__(self, **kwargs)
        self.addFactoryArguments(treename=treename, l10nbuilds=l10nbuilds, cb=cb)
        self.treename = treename
        self.l10nbuilds = l10nbuilds
        self.cb = cb

    def start(self):
        from scheduler import Tree

        loog = self.addLog("stdio")
        self.pending = 0
        properties = self.build.getProperties()
        self.rendered_tree = tree = properties.render(self.treename)
        l10nbuilds = properties.render(self.l10nbuilds)
        cp = ConfigParser()
        cp.read(l10nbuilds)
        repo = cp.get(tree, "repo")
        branch = cp.get(tree, "mozilla")
        path = cp.get(tree, "l10n.ini")
        l10nbranch = cp.get(tree, "l10n")
        locales = cp.get(tree, "locales")
        if locales == "all":
            alllocales = "yes"
        else:
            alllocales = "no"
            properties.update({"locales": filter(None, locales.split())}, "Build")
        self.tree = Tree(self.rendered_tree, repo, branch, l10nbranch, path)
        loog.addStdout("Loading l10n.inis for %s\n" % self.rendered_tree)
        logger.debug(
            "scheduler.l10n.tree", "Loading l10n.inis for %s, alllocales: %s" % (self.rendered_tree, alllocales)
        )
        self.loadIni(repo, branch, path, alllocales)

    def loadIni(self, repo, branch, path, alllocales="no"):
        url = repo + "/" + branch + "/raw-file/default/" + path
        self.getLog("stdio").addStdout("\nloading %s\n" % url)
        self.step_status.setText(["loading", "l10n.ini"])
        self.step_status.setText2([repo, branch, path])
        self.pending += 1
        d = getPage(url)
        d.addCallbacks(self.onL10niniLoad, self.onL10niniFail, callbackArgs=[repo, branch, path, alllocales])

    def onL10niniLoad(self, inicontent, repo, branch, path, alllocales):
        self.pending -= 1
        logger.debug("scheduler.l10n.tree", "Loaded %s, alllocales: %s" % (path, alllocales))
        self.step_status.setText(["loaded", "l10n.ini"])
        loog = self.getLog("stdio")
        cp = ConfigParser()
        cp.readfp(StringIO(inicontent), path)
        try:
            dirs = cp.get("compare", "dirs").split()
        except (NoOptionError, NoSectionError):
            dirs = []
        try:
            dirs += cp.get("extras", "dirs").split()
        except (NoOptionError, NoSectionError):
            pass
        try:
            tld = cp.get("compare", "tld")
            # remove tld from comparison dirs
            if tld in dirs:
                dirs.remove(tld)
        except (NoOptionError, NoSectionError):
            tld = None

        if dirs:
            loog.addStdout("adding %s on branch %s for %s\n" % (", ".join(dirs), branch, self.rendered_tree))
        if tld is not None:
            loog.addStdout("adding a tld compare for %s on %s\n" % (tld, branch))

        self.tree.addData(branch, path, dirs, tld)

        try:
            for title, _path in cp.items("includes"):
                try:
                    # check if the load details are overloaded
                    details = dict(cp.items("include_%s" % title))
                    if details["type"] != "hg":
                        continue
                    loog.addStdout(
                        "need to load %s from %s on %s, %s\n"
                        % (title, details["l10n.ini"], details["repo"], details["mozilla"])
                    )
                    # check if we got the en-US branch already, if not
                    # we're likely loading toolkit off a different repo
                    enbranch = details["mozilla"]
                    if enbranch not in self.tree.branches.values():
                        self.tree.branches[title] = enbranch
                    self.loadIni(details["repo"], details["mozilla"], details["l10n.ini"])
                except NoSectionError:
                    loog.addStdout("need to load %s from %s\n" % (title, _path))
                    self.loadIni(repo, branch, _path)
        except NoSectionError:
            pass
        try:
            if alllocales == "yes":
                allpath = cp.get("general", "all")
                self.tree.all_locales = allpath
                logger.debug("scheduler.l10n.tree", "loading all-locales for %s from %s" % (self.tree.name, allpath))
                self.pending += 1
                d = getPage(repo + "/" + branch + "/raw-file/default/" + allpath)
                d.addCallbacks(self.allLocalesLoaded, self.allLocalesFailed)
        except NoSectionError:
            pass
        self.endLoad()

    def onL10niniFail(self, failure):
        self.pending -= 1
        loog = self.getLog("stdio")
        loog.addStderr(failure.getErrorMessage())
        if self.pending <= 0:
            self.step_status.setText(["configure", self.rendered_tree, "failed"])
            self.step_status.setText2([])
            self.finished(FAILURE)
        return failure

    def allLocalesLoaded(self, page):
        self.pending -= 1
        locales = filter(None, page.splitlines())
        self.build.setProperty("locales", locales, "Build")
        logger.debug("scheduler.l10n.tree", "all-locales loaded, found %s" % str(locales))
        self.endLoad()

    def allLocalesFailed(self, page):
        self.pending -= 1
        if self.pending <= 0:
            self.step_status.setText(["configure", self.rendered_tree, "failed"])
            self.step_status.setText2([])
            self.finished(FAILURE)
        return failure

    def endLoad(self):
        logger.debug("scheduler.l10n.tree", "load ended, pending jobs: %d" % self.pending)
        if self.pending <= 0:
            self.step_status.setText(["configured", self.rendered_tree])
            self.step_status.setText2([])
            if self.cb is not None:
                try:
                    self.tree.locales = self.build.getProperties().getProperty("locales", [])[:]
                    self.cb(self.tree, changes=self.build.allChanges())
                except Exception, e:
                    logger.debug("scheduler.l10n.tree", str(e))
            self.finished(SUCCESS)
Example #4
0
class TreeLoader(BuildStep):
    '''BuildStep to load data from l10n.ini on remote repos.

    Does mostly just async network traffic, directly on the master,
    it wouldn't be more work there if we'd use the slave, and then
    marshall the data through the network back to the master.
    '''
    def __init__(self, treename, l10nbuilds, cb=None, **kwargs):
        '''Create a TreeLoader step. In addition to the standard arguments,
        treename is the name of the tree,
        l10nbuilds is the local ini file describing the builds,
        cb is a callback with signature (tree, changes=None)
        '''
        BuildStep.__init__(self, **kwargs)
        self.addFactoryArguments(treename=treename,
                                 l10nbuilds=l10nbuilds,
                                 cb=cb)
        self.treename = treename
        self.l10nbuilds = l10nbuilds
        self.cb = cb

    def start(self):
        from scheduler import Tree
        loog = self.addLog('stdio')
        self.pending = 0
        properties = self.build.getProperties()
        self.rendered_tree = tree = properties.render(self.treename)
        l10nbuilds = properties.render(self.l10nbuilds)
        cp = ConfigParser()
        cp.read(l10nbuilds)
        repo = cp.get(tree, 'repo')
        branch = cp.get(tree, 'mozilla')
        path = cp.get(tree, 'l10n.ini')
        l10nbranch = cp.get(tree, 'l10n')
        locales = cp.get(tree, 'locales')
        if locales == 'all':
            alllocales = "yes"
        else:
            alllocales = "no"
            properties.update({'locales': filter(None, locales.split())},
                              "Build")
        self.tree = Tree(self.rendered_tree, repo, branch, l10nbranch, path)
        loog.addStdout('Loading l10n.inis for %s\n' % self.rendered_tree)
        logger.debug(
            'scheduler.l10n.tree', 'Loading l10n.inis for %s, alllocales: %s' %
            (self.rendered_tree, alllocales))
        self.loadIni(repo, branch, path, alllocales)

    def loadIni(self, repo, branch, path, alllocales="no"):
        url = repo + '/' + branch + '/raw-file/default/' + path
        self.getLog('stdio').addStdout('\nloading %s\n' % url)
        self.step_status.setText(['loading', 'l10n.ini'])
        self.step_status.setText2([repo, branch, path])
        self.pending += 1
        d = getPage(url)
        d.addCallbacks(self.onL10niniLoad,
                       self.onL10niniFail,
                       callbackArgs=[repo, branch, path, alllocales])

    def onL10niniLoad(self, inicontent, repo, branch, path, alllocales):
        self.pending -= 1
        logger.debug('scheduler.l10n.tree',
                     'Loaded %s, alllocales: %s' % (path, alllocales))
        self.step_status.setText(['loaded', 'l10n.ini'])
        loog = self.getLog('stdio')
        cp = ConfigParser()
        cp.readfp(StringIO(inicontent), path)
        try:
            dirs = cp.get('compare', 'dirs').split()
        except (NoOptionError, NoSectionError):
            dirs = []
        try:
            dirs += cp.get('extras', 'dirs').split()
        except (NoOptionError, NoSectionError):
            pass
        try:
            tld = cp.get('compare', 'tld')
            # remove tld from comparison dirs
            if tld in dirs:
                dirs.remove(tld)
        except (NoOptionError, NoSectionError):
            tld = None

        if dirs:
            loog.addStdout("adding %s on branch %s for %s\n" %
                           (", ".join(dirs), branch, self.rendered_tree))
        if tld is not None:
            loog.addStdout("adding a tld compare for %s on %s\n" %
                           (tld, branch))

        self.tree.addData(branch, path, dirs, tld)

        try:
            for title, _path in cp.items('includes'):
                try:
                    # check if the load details are overloaded
                    details = dict(cp.items('include_%s' % title))
                    if details['type'] != 'hg':
                        continue
                    loog.addStdout("need to load %s from %s on %s, %s\n" %
                                   (title, details['l10n.ini'],
                                    details['repo'], details['mozilla']))
                    # check if we got the en-US branch already, if not
                    # we're likely loading toolkit off a different repo
                    enbranch = details['mozilla']
                    if enbranch not in self.tree.branches.values():
                        self.tree.branches[title] = enbranch
                    self.loadIni(details['repo'], details['mozilla'],
                                 details['l10n.ini'])
                except NoSectionError:
                    loog.addStdout("need to load %s from %s\n" %
                                   (title, _path))
                    self.loadIni(repo, branch, _path)
        except NoSectionError:
            pass
        try:
            if alllocales == 'yes':
                allpath = cp.get('general', 'all')
                self.tree.all_locales = allpath
                logger.debug(
                    'scheduler.l10n.tree',
                    'loading all-locales for %s from %s' %
                    (self.tree.name, allpath))
                self.pending += 1
                d = getPage(repo + '/' + branch + '/raw-file/default/' +
                            allpath)
                d.addCallbacks(self.allLocalesLoaded, self.allLocalesFailed)
        except NoSectionError:
            pass
        self.endLoad()

    def onL10niniFail(self, failure):
        self.pending -= 1
        loog = self.getLog('stdio')
        loog.addStderr(failure.getErrorMessage())
        if self.pending <= 0:
            self.step_status.setText(
                ['configure', self.rendered_tree, 'failed'])
            self.step_status.setText2([])
            self.finished(FAILURE)
        return failure

    def allLocalesLoaded(self, page):
        self.pending -= 1
        locales = util.parseLocales(page)
        self.build.setProperty('locales', locales, 'Build')
        logger.debug('scheduler.l10n.tree',
                     'all-locales loaded, found %s' % str(locales))
        self.endLoad()

    def allLocalesFailed(self, page):
        self.pending -= 1
        if self.pending <= 0:
            self.step_status.setText(
                ['configure', self.rendered_tree, 'failed'])
            self.step_status.setText2([])
            self.finished(FAILURE)
        return failure

    def endLoad(self):
        logger.debug('scheduler.l10n.tree',
                     'load ended, pending jobs: %d' % self.pending)
        if self.pending <= 0:
            self.step_status.setText(['configured', self.rendered_tree])
            self.step_status.setText2([])
            if self.cb is not None:
                try:
                    self.tree.locales = self.build.getProperties().getProperty(
                        'locales', [])[:]
                    self.cb(self.tree, changes=self.build.allChanges())
                except Exception, e:
                    logger.debug('scheduler.l10n.tree', str(e))
            self.finished(SUCCESS)