def __init__(self, buildout, name, options):

        # Rip off p.r.zope2instance's Egg magic to support ``eggs`` parameter
        # for develop eggs, etc.
        self.egg = Egg(buildout, options['recipe'], options)
        self.buildout = buildout
        self.options = options
Ejemplo n.º 2
0
    def __init__(self, buildout, name, options):
        from zc.recipe.egg import Egg

        self.buildout, self.options, self.name = buildout, options, name
        self.egg = Egg(buildout, options['recipe'], options)

        bin_test = os.path.join(buildout['buildout']['bin-directory'], 'test')
        self.testscript = self.options.get('test-script', bin_test)

        if 'eggs' not in self.options:
            # Fallback to [test]'s eggs if available.
            if 'test' in self.buildout:
                self.options['eggs'] = self.buildout['test'].get('eggs')

        self.default_policy = self.options.get('default-policy', 'include').strip()

        self.exclude = self.options.get('exclude', '').split()
        self.exclude_groups = self.options.get('exclude-groups', '').split()
        self.include = self.options.get('include', '').split()
        self.include_groups = self.options.get('include-groups', '').split()

        options['location'] = os.path.join(
            buildout['buildout']['parts-directory'],
            self.name,
        )
class Recipe(object):
    
    def __init__(self, buildout, name, options):

        # Rip off p.r.zope2instance's Egg magic to support ``eggs`` parameter
        # for develop eggs, etc.
        self.egg = Egg(buildout, options['recipe'], options)
        self.buildout = buildout
        self.options = options

    def install(self):

        # Create var dirs
        var = os.path.join(self.buildout['buildout']['directory'], 'var')
        fs = os.path.join(self.buildout['buildout']['directory'], 'var', 'filestorage')
        bs = os.path.join(self.buildout['buildout']['directory'], 'var', 'blobstorage')
        mkdir(var)
        mkdir(fs)
        mkdir(bs)

        # Generate paster script
        requirements, ws = self.egg.working_set(['collective.recipe.bluebream'])
        return scripts(['PasteScript'], ws,
            self.buildout['buildout']['executable'],
            self.buildout['buildout']['bin-directory'])

    def update(self):
        pass
    def __init__(self, buildout, name, options):
        self.egg = Egg(buildout, options['recipe'], options)
        self.buildout, self.name, self.options = buildout, name, options
        options['bin-directory'] = buildout['buildout']['bin-directory']

        if not "instance" in buildout:
            raise ValueError("Buildout missing [instance] section")

        options['eggs'] = buildout['instance']['eggs']
Ejemplo n.º 5
0
    def __init__(self, buildout, name, options):
        self.buildout, self.name, self.options = buildout, name, options
        self.egg = Egg(buildout, options["recipe"], options)

        options.setdefault("bin-directory", buildout["buildout"]["bin-directory"])

        options.setdefault("project", "project")
        options.setdefault("settings", "settings")
        options.setdefault("create-project", "true")
        options.setdefault("url-conf", options["project"] + ".urls")
        options.setdefault("extra-paths", buildout["buildout"].get("extra-paths", ""))
        options.setdefault("script-name", name)
Ejemplo n.º 6
0
 def getWorkingSet(self):
     """If you want do override the default working set"""
     egg = Egg(self.buildout, 'slapos.cookbook', self.options.copy())
     requirements, ws = egg.working_set()
     return ws
    def install(self, update=False):
        """installer
        """

        parts = [self.part_dir, self.var_dir]

        if not update:
            for path in parts:
                if os.path.exists(path):
                    shutil.rmtree(path)

            # Copy the solr files
            shutil.copytree(
                os.path.join(
                    self.options['solr-location'],
                    'example'
                ),
                self.part_dir
            )

        dirs = [
            self.data_dir,
            os.path.join(self.var_dir, 'logs')
        ]

        logdir = dirs[1]

        for path in parts + dirs:
            if not os.path.exists(path):
                os.makedirs(path)

        solrconfig_path = os.path.join(
            self.part_dir,   'solr', 'collection1', 'conf', 'solrconfig.xml'
        )
        if not self.options['solr-config']:
            with open(solrconfig_path, 'rb') as src:
                solrconfig = src.read().replace(
                    'dir="../..',
                    'dir="%s' % self.options['solr-location']
                )
            with open(solrconfig_path, 'wb') as dst:
                dst.write(solrconfig)
        else:
            shutil.copy(self.options['solr-config'], solrconfig_path)

        eggs = Egg(self.buildout, self.options['recipe'], self.options)
        __, working_set = eggs.working_set(
            self.eggs
        )

        # The initialization code is expressed as a list of lines
        initialization = []

        # Gets the initialization code: the tricky part here is to preserve
        # indentation. This is obtained by getting all the lines and then
        # finding the initial whitespace common to all lines, excluding blank
        # lines: the obtained whitespace is then subtracted from all lines.
        raw_code = []
        residual_whitespace = None
        whitespace_regex = re.compile(r'^([ ]+)')
        for line in self.options.get("initialization", "").splitlines():
            if line != "":
                m = whitespace_regex.search(line)
                if m is None:
                    initial_whitespace = 0
                else:
                    initial_whitespace = len(m.group(1))
                if residual_whitespace is None or \
                        initial_whitespace < residual_whitespace:
                    residual_whitespace = initial_whitespace
                raw_code.append(line)
        for line in raw_code:
            initialization.append(line[residual_whitespace:])

        # Gets the environment-vars option and generates code to set the
        # enviroment variables via os.environ
        environment_vars = []
        for line in self.listify(self.options.get("environment-vars", "")):
            try:
                var_name, raw_value = line.split(" ", 1)
            except ValueError:
                raise RuntimeError(
                    "Bad environment-vars contents: %s" % line
                )
            environment_vars.append(
                'os.environ["%s"] = r"%s"' % (
                    var_name,
                    raw_value.strip()
                )
            )
        if len(environment_vars) > 0:
            initialization.append("import os")
            initialization.extend(environment_vars)

        script_file = os.path.join(self.buildout['buildout']['bin-directory'],
                                   self.name)
        schema_file = os.path.join(
                self.part_dir,
                'solr',
                'collection1',
                'conf',
                'schema.xml'
            )
        if os.path.exists(schema_file):
            os.remove(schema_file)
        self.generate(
            src=os.path.join(TEMPLATE_DIR, 'solr.tmpl'),
            dst=script_file,
            schema_file=schema_file,
            executable=self.buildout['buildout']['executable'],
            extrapaths=[dist.location for dist in working_set],
            otherpaths=self.options.get('extra-paths', '').split(),
            pidfile=os.path.join(self.var_dir, 'solr.pid'),
            logfile=os.path.join(logdir, 'solr.log'),
            buildoutdir=self.buildout['buildout']['directory'],
            basedir=self.part_dir,
            djangosettings=self.options['django-settings'],
            startcmd=self.parse_java_opts(),
            initialization=initialization
        )

        os.chmod(
            script_file,
            stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
        )

        # returns installed files
        return parts
Ejemplo n.º 8
0
    def install(self):
        """Installer"""
        files = []
        options = dict([(k, v) for k, v in self.options.items()])
        options.pop('recipe')

        # creates the dir
        if not os.path.exists(self.location):
            os.mkdir(self.location)

        # static files
        public_html = os.path.join(self.location, 'public_html')
        if not os.path.isdir(public_html):
            os.mkdir(public_html)

        for filename in ('index.html', 'classic.css', 'robots.txt'):
            if filename == 'classic.css':
                destination = os.path.join(public_html, 'buildbot.css')
            else:
                destination = os.path.join(public_html, filename)
            shutil.copyfile(os.path.join(self.public_html, filename),
                            destination)
            files.append(destination)

        if 'public-html' in options:
            dirname = options.pop('public-html')
            for filename in os.listdir(dirname):
                if filename.startswith('.'):
                    continue
                destination = os.path.join(public_html, filename)
                shutil.copyfile(os.path.join(dirname, filename), destination)
                if destination not in files:
                    files.append(destination)

        # virtual env
        self.create_virtualenv(self.location)

        # adds buildbot.tac
        template = open(join(self.recipe_dir, 'buildbot.tac_tmpl')).read()
        template = template % {'base_dir': self.location}
        buildbot_tac = join(self.location, 'buildbot.tac')
        open(buildbot_tac, 'w').write(str(template))
        self.log('Generated script %r.' % buildbot_tac)
        files.append(buildbot_tac)

        # Create an empty log file if necessary to avoid the error
        # message on first run.
        if not os.path.exists(os.path.join(self.location, 'twistd.log')):
            open(os.path.join(self.location, 'twistd.log'), 'w').write('')


        # generates the buildbot.cfg file
        
        # JKM: Instead of pulling slaves from the master config file, 
        # read 'em from slaves.cfg. This lets us check everything else
        # into SCM and just ignore slaves.cfg
        if os.path.exists('slaves.cfg'):
            slave_config = ConfigParser()
            slave_config.read('slaves.cfg')
            slaves = dict(slave_config.items('slaves'))
        elif 'slaves' in options:
            slaves = options.pop('slaves')
            slaves = dict([slave.split()[:2]
                           for slave in slaves.splitlines()
                           if slave.strip() != ''])
        else:
            slaves = {}

        parts_directory = join(self.buildout['buildout']['parts-directory'])
        for k in ( 'projects', 'pollers'):
            options.setdefault('%s-directory' % k, join(parts_directory, k))

        for k, v in (('port', '8999'), ('wport', '9000'),
                     ('project-name', 'Buildbot'),
                     ('allow-force', 'false')):
            options.setdefault(k, v)

        for k, v in (('url', 'http://localhost:%s/'),
                     ('project-url', 'http://localhost:%s/')):
            options.setdefault(k, v % options['wport'])

        # assume trailing slash
        for k in ('url', 'project-url'):
            url = options[k]
	    if not url.endswith('/'):
                options[k] = url + '/'	

        globs = dict(buildbot=options,
                     slaves=slaves)

        files.append(self.write_config('buildbot', **globs))

        # generate script
        # JKM: use my package name instead of collective.buildbot
        options = {'eggs':'django_buildbot_recipes',
                   'entry-points': '%s=collective.buildbot.scripts:main' % self.name,
                   'arguments': 'location=%r, config_file=%r' % (
                       self.location, join(self.location, 'buildbot.cfg'))
                  }
        script = Egg(self.buildout, self.name, options)
        files.extend(list(script.install()))

        return files
class Recipe(object):
    """zc.buildout recipe"""

    def __init__(self, buildout, name, options):
        self.egg = Egg(buildout, options['recipe'], options)
        self.buildout, self.name, self.options = buildout, name, options
        options['bin-directory'] = buildout['buildout']['bin-directory']

        if not "instance" in buildout:
            raise ValueError("Buildout missing [instance] section")

        options['eggs'] = buildout['instance']['eggs']

    def install_developer_manual(self):
        # First try to pull existing developermanual wc
        print "About to start installing the collective.developermanual"

        git_url = "git://github.com/collective/collective.developermanual.git"
        location = self.options.get("dev-manual-location", None)
        cwd = os.getcwd()
        buildout_dir = self.buildout["buildout"]["directory"]
        os.chdir(buildout_dir)

        if not location:
            location = os.path.join(buildout_dir, "developermanual")

        if os.path.exists(location):
            os.chdir(location)
            exists = True
        else:
            parent = os.path.dirname(location)
            if not os.path.exists(parent):
                try:
                    os.makedirs(parent)
                except OSError:
                    print "You might not have permission to create %s" % parent
                    sys.exit(1)

            exists = False

        kwargs = {}
        kwargs['stdout'] = subprocess.PIPE
        kwargs['stderr'] = subprocess.PIPE

        if exists:
            # developermanual was already cloned, need to pull
            print ("We seem to have an existing manual in %s, let's try to "
                   "update it" % os.getcwd())
            cmd = ['git','pull','--quiet']
            res = subprocess.Popen(cmd, **kwargs)
            stdout, stderr = res.communicate()
            if stdout or stderr:
                # This doesn't seem to be a git repo. Let's remove it
                # and try to clone it
                print "This was not a valid directory, removing..."
                exists = False
                os.chdir(buildout_dir)
                os.rmdir(location)
            else:
                print "Manual updated successfuly"

        if not exists:
            print "We need to do a fresh checkout, this might take a while..."
            os.chdir(parent)
            dirname = os.path.basename(location)
            cmd = ['git','clone', "--quiet", git_url, dirname]
            res = subprocess.Popen(cmd, **kwargs)
            stdout, stderr = res.communicate()
            if stdout or stderr:
                print ("Something went wrong, please check the error and "
                       "try again.")
                print stdout
                print stderr
                sys.exit(1)

            else:
                os.chdir(location)
                print "Done. Manual located in %s" % os.getcwd()

        print "Bootstrapping..."
        cmd = ['python', 'bootstrap.py']
        res = subprocess.Popen(cmd, **kwargs)
        stdout, stderr = res.communicate()

        print "Running buildout. This might take a while..."
        cmd = ['./bin/buildout']
        res = subprocess.Popen(cmd, **kwargs)
        stdout, stderr = res.communicate()

        print "Generating doc. This might take a while..."
        cmd = ['make', 'html']
        res = subprocess.Popen(cmd, **kwargs)
        stdout, stderr = res.communicate()

        self.developer_manual_uri = os.path.join(location, 'build/html')

        os.chdir(cwd)

        print "All Done."

    def create_ploneide_conf(self):
        # from here, we will create a ploneide.conf file inside ploneide's
        # directory, in where we'll store different configuration parameters
        # needed to modify the IDE's behavior
        buildout_dir = self.buildout["buildout"]["directory"]
        location = self.buildout["instance"]["location"]

        zope_conf_path = os.path.join(location, "etc", "zope.conf")

        instance_host = self.options.get("instance-host", None)

        if not instance_host:
            instance_host = self.buildout["instance"]["http-address"]
            if len(instance_host.split(':')) > 1:
                instance_host = instance_host.split(':')[0]
            else:
                instance_host = 'localhost'


        instance_port = self.options.get("instance-port", None)

        if not instance_port:
            instance_port = self.buildout["instance"]["http-address"]
            if len(instance_port.split(':')) > 1:
                instance_port = instance_port.split(':')[1]

        instance_port = int(instance_port)

        ploneide_host = self.options.get("ploneide-host", instance_host)
        ploneide_port = int(self.options.get("ploneide-port", None) or
                            instance_port + 100)

        debug_host = self.options.get("debug-host", ploneide_host)
        debug_port = int(self.options.get("debug-port", None) or
                            ploneide_port + 1)

        dirs = self.options.get("directories", None)
        if dirs:
            devel_directories = dict([i.split(' ') for i in dirs.split('\n')
                                      if i!= ''])
        else:
            devel_directories = {'src': os.path.join(buildout_dir,'src')}

        config = ConfigParser.RawConfigParser()

        config.add_section('Directories')
        config.add_section('Servers')
        config.add_section('Dev Manual')

        config.set('Directories', 'buildout', buildout_dir)
        config.set('Directories', 'devel', devel_directories)
        config.set('Directories', 'zope-conf-file', zope_conf_path)

        config.set('Servers', 'instance-host', instance_host)
        config.set('Servers', 'instance-port', instance_port)
        config.set('Servers', 'ploneide-host', ploneide_host)
        config.set('Servers', 'ploneide-port', ploneide_port)
        config.set('Servers', 'debug-host', debug_host)
        config.set('Servers', 'debug-port', debug_port)

        config.set('Dev Manual', 'location', self.developer_manual_uri)

        config_file = os.path.join(buildout_dir,'ploneide.conf')
        with open(config_file, 'wb') as configfile:
            config.write(configfile)

        return config_file


    def install(self):
        """Installer"""
        # XXX Implement recipe functionality here
        options = self.options

        self.developer_manual_uri = ""

        install_dev_manual = self.options.get("dev-manual", "local")

        if install_dev_manual == "local":
            self.install_developer_manual()
        else:
            print "Using online collective.developermanual"
            self.developer_manual_uri = "http://developer.plone.org"

        config_file = self.create_ploneide_conf()

        requirements, ws = self.egg.working_set(['collective.recipe.ploneide'])
        zc.buildout.easy_install.scripts(
            [(options.get('control-script', self.name),
              'collective.ploneide.ploneide.main', 'main')],
            ws, options['executable'], options['bin-directory'],
            arguments = ('\n        %r' % config_file),
            )

        # Return files that were created by the recipe. The buildout
        # will remove all returned files upon reinstall.
        return tuple()

    def update(self):
        """Updater"""
        pass
Ejemplo n.º 10
0
class Recipe(object):
    def __init__(self, buildout, name, options):
        self.buildout, self.name, self.options = buildout, name, options
        self.egg = Egg(buildout, options["recipe"], options)

        options.setdefault("bin-directory", buildout["buildout"]["bin-directory"])

        options.setdefault("project", "project")
        options.setdefault("settings", "settings")
        options.setdefault("create-project", "true")
        options.setdefault("url-conf", options["project"] + ".urls")
        options.setdefault("extra-paths", buildout["buildout"].get("extra-paths", ""))
        options.setdefault("script-name", name)

    def install(self):
        scripts = []
        base_dir = self.buildout["buildout"]["directory"]
        extra_paths = [base_dir]

        extra_paths.extend([p.replace("/", os.path.sep) for p in self.options["extra-paths"].splitlines() if p.strip()])

        for p in os.listdir(self.buildout["buildout"]["eggs-directory"]):
            if p.endswith(".egg"):
                extra_paths.append(os.path.join(base_dir, ".duke/eggs/%s/" % p))

        requirements, ws = self.egg.working_set(["djangodukerecipe"])

        # Create the Django management script
        scripts.extend(self.create_manage_script(extra_paths, ws))

        # Make the wsgi and fastcgi scripts if enabled
        # scripts.extend(self.make_scripts(extra_paths, ws))

        if self.options["create-project"] == "true":
            self.create_project()

        return scripts

    def create_manage_script(self, extra_paths, ws):
        return zc.buildout.easy_install.scripts(
            [(self.options["script-name"], "djangodukerecipe.manage", "main")],
            ws,
            self.options["executable"],
            self.options["bin-directory"],
            extra_paths=extra_paths,
            arguments='"%s"' % self.options["settings"],
        )

    def create_project(self):
        """
        Django 1.4 project structure

        + project-root-folder/
          - manage.py (skipped)
          + project
            - __init__.py
            - settings.py
            - urls.py
            - wsgi.py (skipped)

        Django duke final project structure

        + project-root-folder/
          - setup.py
          - bootstrap.py
          - buildout.cfg
          + project
            - __init__.py
            - settings.py
            - local_settings.py
            - urls.py
            + conf/
              - default.py
              - dev.py
        """
        base_dir = self.buildout["buildout"]["directory"]
        project_dir = os.path.join(base_dir, self.options["project"])
        bin_path = os.path.join(base_dir, ".duke/bin/")
        tmp_path = os.path.join("/tmp/", self.options["project"])

        if os.path.exists(project_dir):
            logger.info("Skipping creating of project: %(project)s " "since it exists" % self.options)
        else:
            # p = "',\n    '".join(self.buildout['python']['extra-paths'].split('\n'))

            logger.info("Creating project: %s " % self.options["project"])
            logger.info(self.options["extra-paths"])

            if self.options.get("template"):
                options = "--template=%s --extension=py,rst,md,default" % self.options["template"]
            else:
                options = ""

            # Lazy fix for weird file permission problem..
            self.command("chmod a+x " + os.path.join(bin_path, "django"))

            django_admin = None
            for d in os.listdir(self.buildout["buildout"]["eggs-directory"]):
                p = os.path.join(self.buildout["buildout"]["eggs-directory"], d)
                if d.startswith("Django") and os.path.exists(os.path.join(p, "django/bin/django-admin.py")):
                    django_admin = os.path.join(p, "django/bin/django-admin.py")

            self.command(
                "%(django)s startproject %(options)s %(project)s %(dest)s"
                % {
                    "django": django_admin or os.path.join(bin_path, "django"),
                    "project": self.options["project"],
                    "options": options,
                    "dest": base_dir,
                }
            )

    # def make_scripts(self, extra_paths, ws):
    #    # The scripts function uses a script_template variable hardcoded
    #    # in Buildout to generate the script file. Since the wsgi file
    #    # needs to create a callable application function rather than call
    #    # a script, monkeypatch the script template here.
    #    _script_template = zc.buildout.easy_install.script_template

    #    zc.buildout.easy_install.script_template = \
    #        zc.buildout.easy_install.script_header + WSGI_TEMPLATE

    #    generated = zc.buildout.easy_install.scripts(
    #        [('%s.wsgi' % self.options['script-name'],
    #            'djangodukerecipe.wsgi', 'main')],
    #        ws, self.options['executable'],
    #        self.options['bin-directory'], extra_paths = extra_paths,
    #        #arguments= "'%s.%s'" % (self.options["project"],
    #        #    self.options['settings'])
    #        arguments = "'%s'" % self.options.get("settings", self.options.get('project') + ".settings")
    #    )

    #    zc.buildout.easy_install.script_template = _script_template

    #    return generated

    def update(self):
        self.install()

    def command(self, cmd, **kwargs):
        logger.info("Executing %s " % cmd)
        output = subprocess.PIPE
        if self.buildout["buildout"].get("verbosity"):
            output = None
        command = subprocess.Popen(cmd, shell=True, stdout=output, **kwargs)
        return command.wait()
Ejemplo n.º 11
0
 def setup_working_set(self):
     egg = Egg(self.buildout, 'Django', self.options)
     self.working_set = egg.working_set(self.eggs)
Ejemplo n.º 12
0
 def setup_working_set(self):
     egg = Egg(
         self.buildout, 'Django', self.options
     )
     self.working_set = egg.working_set(self.eggs)
Ejemplo n.º 13
0
class Recipe(object):

    def __init__(self, buildout, name, options):
        self.buildout, self.options, self.name = buildout, options, name
        self.egg = Egg(buildout, options['recipe'], options)

        bin_test = os.path.join(buildout['buildout']['bin-directory'], 'test')
        self.testscript = self.options.get('test-script', bin_test)

        if 'eggs' not in self.options:
            # Fallback to [test]'s eggs if available.
            if 'test' in self.buildout:
                self.options['eggs'] = self.buildout['test'].get('eggs')

        self.default_policy = self.options.get('default-policy', 'include').strip()

        self.exclude = self.options.get('exclude', '').split()
        self.exclude_groups = self.options.get('exclude-groups', '').split()
        self.include = self.options.get('include', '').split()
        self.include_groups = self.options.get('include-groups', '').split()

        options['location'] = os.path.join(
            buildout['buildout']['parts-directory'],
            self.name,
        )

    def install(self):
        options = self.options
        location = options['location']

        reqs, ws = self.egg.working_set(['plone.recipe.alltests'])

        packages = []
        paths = {}
        for dist in ws.by_key.values():
            name = dist.project_name
            packages.append(name)
            paths[name] = dist.location

        excludes = [re.compile(e) for e in self.exclude]
        includes = [re.compile(e) for e in self.include]

        filtered_packages = []
        for p in list(set(packages) - EXCLUDE_PACKAGES):
            match = False
            if self.default_policy == 'include':
                for e in excludes:
                    if e.search(p) is not None:
                        match = True
                        break
                if not match:
                    filtered_packages.append(p)
            elif self.default_policy == 'exclude':
                for i in includes:
                    if i.search(p) is not None:
                        match = True
                        break
                if match:
                    filtered_packages.append(p)
        packages = filtered_packages

        # Allow to map distribution names to different package names
        pmap = dict()
        package_map = options.get('package-map', '').strip()
        if package_map:
            pmap = self.buildout[package_map]

        for k, v in pmap.items():
            if k in packages:
                packages.remove(k)
                packages.append(v)
                paths[v] = paths[k]
                del paths[k]
        packages.sort()

        # Allow to group multiple packages into one test run
        groups = dict()
        groups_section = options.get('groups', '').strip()
        if groups_section:
            data = self.buildout[groups_section]
            for k, v in data.items():
                if self.default_policy == 'include':
                    if k in self.exclude_groups:
                        for p in v.split():
                            if p in packages:
                                packages.remove(p)
                    else:
                        groups[k] = v.split()
                elif self.default_policy == 'exclude' and k in self.include_groups:
                    groups[k] = v.split()

        easy_install.scripts(
            [(self.name, 'plone.recipe.alltests.runner', 'main')],
            ws, options['executable'], options['bin-directory'],
            arguments=dict(
                packages=packages,
                testscript=self.testscript,
                paths=paths,
                groups=groups,
            ),
        )

        return location

    def update(self):
        return self.install()