def __init__(self, buildout, name, options):
# Rip off p.r.zope2instance's Egg magic to support ``eggs`` parameter
# for develop eggs, etc.
self.egg = Egg(buildout, options['recipe'], options)
self.buildout = buildout
self.options = options
def __init__(self, buildout, name, options):
from zc.recipe.egg import Egg
self.buildout, self.options, self.name = buildout, options, name
self.egg = Egg(buildout, options['recipe'], options)
bin_test = os.path.join(buildout['buildout']['bin-directory'], 'test')
self.testscript = self.options.get('test-script', bin_test)
if 'eggs' not in self.options:
# Fallback to [test]'s eggs if available.
if 'test' in self.buildout:
self.options['eggs'] = self.buildout['test'].get('eggs')
self.default_policy = self.options.get('default-policy', 'include').strip()
self.exclude = self.options.get('exclude', '').split()
self.exclude_groups = self.options.get('exclude-groups', '').split()
self.include = self.options.get('include', '').split()
self.include_groups = self.options.get('include-groups', '').split()
options['location'] = os.path.join(
buildout['buildout']['parts-directory'],
self.name,
)
class Recipe(object):
def __init__(self, buildout, name, options):
# Rip off p.r.zope2instance's Egg magic to support ``eggs`` parameter
# for develop eggs, etc.
self.egg = Egg(buildout, options['recipe'], options)
self.buildout = buildout
self.options = options
def install(self):
# Create var dirs
var = os.path.join(self.buildout['buildout']['directory'], 'var')
fs = os.path.join(self.buildout['buildout']['directory'], 'var', 'filestorage')
bs = os.path.join(self.buildout['buildout']['directory'], 'var', 'blobstorage')
mkdir(var)
mkdir(fs)
mkdir(bs)
# Generate paster script
requirements, ws = self.egg.working_set(['collective.recipe.bluebream'])
return scripts(['PasteScript'], ws,
self.buildout['buildout']['executable'],
self.buildout['buildout']['bin-directory'])
def update(self):
pass
def __init__(self, buildout, name, options):
self.egg = Egg(buildout, options['recipe'], options)
self.buildout, self.name, self.options = buildout, name, options
options['bin-directory'] = buildout['buildout']['bin-directory']
if not "instance" in buildout:
raise ValueError("Buildout missing [instance] section")
options['eggs'] = buildout['instance']['eggs']
def __init__(self, buildout, name, options):
self.buildout, self.name, self.options = buildout, name, options
self.egg = Egg(buildout, options["recipe"], options)
options.setdefault("bin-directory", buildout["buildout"]["bin-directory"])
options.setdefault("project", "project")
options.setdefault("settings", "settings")
options.setdefault("create-project", "true")
options.setdefault("url-conf", options["project"] + ".urls")
options.setdefault("extra-paths", buildout["buildout"].get("extra-paths", ""))
options.setdefault("script-name", name)
def getWorkingSet(self):
"""If you want do override the default working set"""
egg = Egg(self.buildout, 'slapos.cookbook', self.options.copy())
requirements, ws = egg.working_set()
return ws
def install(self, update=False):
"""installer
"""
parts = [self.part_dir, self.var_dir]
if not update:
for path in parts:
if os.path.exists(path):
shutil.rmtree(path)
# Copy the solr files
shutil.copytree(
os.path.join(
self.options['solr-location'],
'example'
),
self.part_dir
)
dirs = [
self.data_dir,
os.path.join(self.var_dir, 'logs')
]
logdir = dirs[1]
for path in parts + dirs:
if not os.path.exists(path):
os.makedirs(path)
solrconfig_path = os.path.join(
self.part_dir, 'solr', 'collection1', 'conf', 'solrconfig.xml'
)
if not self.options['solr-config']:
with open(solrconfig_path, 'rb') as src:
solrconfig = src.read().replace(
'dir="../..',
'dir="%s' % self.options['solr-location']
)
with open(solrconfig_path, 'wb') as dst:
dst.write(solrconfig)
else:
shutil.copy(self.options['solr-config'], solrconfig_path)
eggs = Egg(self.buildout, self.options['recipe'], self.options)
__, working_set = eggs.working_set(
self.eggs
)
# The initialization code is expressed as a list of lines
initialization = []
# Gets the initialization code: the tricky part here is to preserve
# indentation. This is obtained by getting all the lines and then
# finding the initial whitespace common to all lines, excluding blank
# lines: the obtained whitespace is then subtracted from all lines.
raw_code = []
residual_whitespace = None
whitespace_regex = re.compile(r'^([ ]+)')
for line in self.options.get("initialization", "").splitlines():
if line != "":
m = whitespace_regex.search(line)
if m is None:
initial_whitespace = 0
else:
initial_whitespace = len(m.group(1))
if residual_whitespace is None or \
initial_whitespace < residual_whitespace:
residual_whitespace = initial_whitespace
raw_code.append(line)
for line in raw_code:
initialization.append(line[residual_whitespace:])
# Gets the environment-vars option and generates code to set the
# enviroment variables via os.environ
environment_vars = []
for line in self.listify(self.options.get("environment-vars", "")):
try:
var_name, raw_value = line.split(" ", 1)
except ValueError:
raise RuntimeError(
"Bad environment-vars contents: %s" % line
)
environment_vars.append(
'os.environ["%s"] = r"%s"' % (
var_name,
raw_value.strip()
)
)
if len(environment_vars) > 0:
initialization.append("import os")
initialization.extend(environment_vars)
script_file = os.path.join(self.buildout['buildout']['bin-directory'],
self.name)
schema_file = os.path.join(
self.part_dir,
'solr',
'collection1',
'conf',
'schema.xml'
)
if os.path.exists(schema_file):
os.remove(schema_file)
self.generate(
src=os.path.join(TEMPLATE_DIR, 'solr.tmpl'),
dst=script_file,
schema_file=schema_file,
executable=self.buildout['buildout']['executable'],
extrapaths=[dist.location for dist in working_set],
otherpaths=self.options.get('extra-paths', '').split(),
pidfile=os.path.join(self.var_dir, 'solr.pid'),
logfile=os.path.join(logdir, 'solr.log'),
buildoutdir=self.buildout['buildout']['directory'],
basedir=self.part_dir,
djangosettings=self.options['django-settings'],
startcmd=self.parse_java_opts(),
initialization=initialization
)
os.chmod(
script_file,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH
)
# returns installed files
return parts
def install(self):
"""Installer"""
files = []
options = dict([(k, v) for k, v in self.options.items()])
options.pop('recipe')
# creates the dir
if not os.path.exists(self.location):
os.mkdir(self.location)
# static files
public_html = os.path.join(self.location, 'public_html')
if not os.path.isdir(public_html):
os.mkdir(public_html)
for filename in ('index.html', 'classic.css', 'robots.txt'):
if filename == 'classic.css':
destination = os.path.join(public_html, 'buildbot.css')
else:
destination = os.path.join(public_html, filename)
shutil.copyfile(os.path.join(self.public_html, filename),
destination)
files.append(destination)
if 'public-html' in options:
dirname = options.pop('public-html')
for filename in os.listdir(dirname):
if filename.startswith('.'):
continue
destination = os.path.join(public_html, filename)
shutil.copyfile(os.path.join(dirname, filename), destination)
if destination not in files:
files.append(destination)
# virtual env
self.create_virtualenv(self.location)
# adds buildbot.tac
template = open(join(self.recipe_dir, 'buildbot.tac_tmpl')).read()
template = template % {'base_dir': self.location}
buildbot_tac = join(self.location, 'buildbot.tac')
open(buildbot_tac, 'w').write(str(template))
self.log('Generated script %r.' % buildbot_tac)
files.append(buildbot_tac)
# Create an empty log file if necessary to avoid the error
# message on first run.
if not os.path.exists(os.path.join(self.location, 'twistd.log')):
open(os.path.join(self.location, 'twistd.log'), 'w').write('')
# generates the buildbot.cfg file
# JKM: Instead of pulling slaves from the master config file,
# read 'em from slaves.cfg. This lets us check everything else
# into SCM and just ignore slaves.cfg
if os.path.exists('slaves.cfg'):
slave_config = ConfigParser()
slave_config.read('slaves.cfg')
slaves = dict(slave_config.items('slaves'))
elif 'slaves' in options:
slaves = options.pop('slaves')
slaves = dict([slave.split()[:2]
for slave in slaves.splitlines()
if slave.strip() != ''])
else:
slaves = {}
parts_directory = join(self.buildout['buildout']['parts-directory'])
for k in ( 'projects', 'pollers'):
options.setdefault('%s-directory' % k, join(parts_directory, k))
for k, v in (('port', '8999'), ('wport', '9000'),
('project-name', 'Buildbot'),
('allow-force', 'false')):
options.setdefault(k, v)
for k, v in (('url', 'http://localhost:%s/'),
('project-url', 'http://localhost:%s/')):
options.setdefault(k, v % options['wport'])
# assume trailing slash
for k in ('url', 'project-url'):
url = options[k]
if not url.endswith('/'):
options[k] = url + '/'
globs = dict(buildbot=options,
slaves=slaves)
files.append(self.write_config('buildbot', **globs))
# generate script
# JKM: use my package name instead of collective.buildbot
options = {'eggs':'django_buildbot_recipes',
'entry-points': '%s=collective.buildbot.scripts:main' % self.name,
'arguments': 'location=%r, config_file=%r' % (
self.location, join(self.location, 'buildbot.cfg'))
}
script = Egg(self.buildout, self.name, options)
files.extend(list(script.install()))
return files
class Recipe(object):
"""zc.buildout recipe"""
def __init__(self, buildout, name, options):
self.egg = Egg(buildout, options['recipe'], options)
self.buildout, self.name, self.options = buildout, name, options
options['bin-directory'] = buildout['buildout']['bin-directory']
if not "instance" in buildout:
raise ValueError("Buildout missing [instance] section")
options['eggs'] = buildout['instance']['eggs']
def install_developer_manual(self):
# First try to pull existing developermanual wc
print "About to start installing the collective.developermanual"
git_url = "git://github.com/collective/collective.developermanual.git"
location = self.options.get("dev-manual-location", None)
cwd = os.getcwd()
buildout_dir = self.buildout["buildout"]["directory"]
os.chdir(buildout_dir)
if not location:
location = os.path.join(buildout_dir, "developermanual")
if os.path.exists(location):
os.chdir(location)
exists = True
else:
parent = os.path.dirname(location)
if not os.path.exists(parent):
try:
os.makedirs(parent)
except OSError:
print "You might not have permission to create %s" % parent
sys.exit(1)
exists = False
kwargs = {}
kwargs['stdout'] = subprocess.PIPE
kwargs['stderr'] = subprocess.PIPE
if exists:
# developermanual was already cloned, need to pull
print ("We seem to have an existing manual in %s, let's try to "
"update it" % os.getcwd())
cmd = ['git','pull','--quiet']
res = subprocess.Popen(cmd, **kwargs)
stdout, stderr = res.communicate()
if stdout or stderr:
# This doesn't seem to be a git repo. Let's remove it
# and try to clone it
print "This was not a valid directory, removing..."
exists = False
os.chdir(buildout_dir)
os.rmdir(location)
else:
print "Manual updated successfuly"
if not exists:
print "We need to do a fresh checkout, this might take a while..."
os.chdir(parent)
dirname = os.path.basename(location)
cmd = ['git','clone', "--quiet", git_url, dirname]
res = subprocess.Popen(cmd, **kwargs)
stdout, stderr = res.communicate()
if stdout or stderr:
print ("Something went wrong, please check the error and "
"try again.")
print stdout
print stderr
sys.exit(1)
else:
os.chdir(location)
print "Done. Manual located in %s" % os.getcwd()
print "Bootstrapping..."
cmd = ['python', 'bootstrap.py']
res = subprocess.Popen(cmd, **kwargs)
stdout, stderr = res.communicate()
print "Running buildout. This might take a while..."
cmd = ['./bin/buildout']
res = subprocess.Popen(cmd, **kwargs)
stdout, stderr = res.communicate()
print "Generating doc. This might take a while..."
cmd = ['make', 'html']
res = subprocess.Popen(cmd, **kwargs)
stdout, stderr = res.communicate()
self.developer_manual_uri = os.path.join(location, 'build/html')
os.chdir(cwd)
print "All Done."
def create_ploneide_conf(self):
# from here, we will create a ploneide.conf file inside ploneide's
# directory, in where we'll store different configuration parameters
# needed to modify the IDE's behavior
buildout_dir = self.buildout["buildout"]["directory"]
location = self.buildout["instance"]["location"]
zope_conf_path = os.path.join(location, "etc", "zope.conf")
instance_host = self.options.get("instance-host", None)
if not instance_host:
instance_host = self.buildout["instance"]["http-address"]
if len(instance_host.split(':')) > 1:
instance_host = instance_host.split(':')[0]
else:
instance_host = 'localhost'
instance_port = self.options.get("instance-port", None)
if not instance_port:
instance_port = self.buildout["instance"]["http-address"]
if len(instance_port.split(':')) > 1:
instance_port = instance_port.split(':')[1]
instance_port = int(instance_port)
ploneide_host = self.options.get("ploneide-host", instance_host)
ploneide_port = int(self.options.get("ploneide-port", None) or
instance_port + 100)
debug_host = self.options.get("debug-host", ploneide_host)
debug_port = int(self.options.get("debug-port", None) or
ploneide_port + 1)
dirs = self.options.get("directories", None)
if dirs:
devel_directories = dict([i.split(' ') for i in dirs.split('\n')
if i!= ''])
else:
devel_directories = {'src': os.path.join(buildout_dir,'src')}
config = ConfigParser.RawConfigParser()
config.add_section('Directories')
config.add_section('Servers')
config.add_section('Dev Manual')
config.set('Directories', 'buildout', buildout_dir)
config.set('Directories', 'devel', devel_directories)
config.set('Directories', 'zope-conf-file', zope_conf_path)
config.set('Servers', 'instance-host', instance_host)
config.set('Servers', 'instance-port', instance_port)
config.set('Servers', 'ploneide-host', ploneide_host)
config.set('Servers', 'ploneide-port', ploneide_port)
config.set('Servers', 'debug-host', debug_host)
config.set('Servers', 'debug-port', debug_port)
config.set('Dev Manual', 'location', self.developer_manual_uri)
config_file = os.path.join(buildout_dir,'ploneide.conf')
with open(config_file, 'wb') as configfile:
config.write(configfile)
return config_file
def install(self):
"""Installer"""
# XXX Implement recipe functionality here
options = self.options
self.developer_manual_uri = ""
install_dev_manual = self.options.get("dev-manual", "local")
if install_dev_manual == "local":
self.install_developer_manual()
else:
print "Using online collective.developermanual"
self.developer_manual_uri = "http://developer.plone.org"
config_file = self.create_ploneide_conf()
requirements, ws = self.egg.working_set(['collective.recipe.ploneide'])
zc.buildout.easy_install.scripts(
[(options.get('control-script', self.name),
'collective.ploneide.ploneide.main', 'main')],
ws, options['executable'], options['bin-directory'],
arguments = ('\n %r' % config_file),
)
# Return files that were created by the recipe. The buildout
# will remove all returned files upon reinstall.
return tuple()
def update(self):
"""Updater"""
pass
class Recipe(object):
def __init__(self, buildout, name, options):
self.buildout, self.name, self.options = buildout, name, options
self.egg = Egg(buildout, options["recipe"], options)
options.setdefault("bin-directory", buildout["buildout"]["bin-directory"])
options.setdefault("project", "project")
options.setdefault("settings", "settings")
options.setdefault("create-project", "true")
options.setdefault("url-conf", options["project"] + ".urls")
options.setdefault("extra-paths", buildout["buildout"].get("extra-paths", ""))
options.setdefault("script-name", name)
def install(self):
scripts = []
base_dir = self.buildout["buildout"]["directory"]
extra_paths = [base_dir]
extra_paths.extend([p.replace("/", os.path.sep) for p in self.options["extra-paths"].splitlines() if p.strip()])
for p in os.listdir(self.buildout["buildout"]["eggs-directory"]):
if p.endswith(".egg"):
extra_paths.append(os.path.join(base_dir, ".duke/eggs/%s/" % p))
requirements, ws = self.egg.working_set(["djangodukerecipe"])
# Create the Django management script
scripts.extend(self.create_manage_script(extra_paths, ws))
# Make the wsgi and fastcgi scripts if enabled
# scripts.extend(self.make_scripts(extra_paths, ws))
if self.options["create-project"] == "true":
self.create_project()
return scripts
def create_manage_script(self, extra_paths, ws):
return zc.buildout.easy_install.scripts(
[(self.options["script-name"], "djangodukerecipe.manage", "main")],
ws,
self.options["executable"],
self.options["bin-directory"],
extra_paths=extra_paths,
arguments='"%s"' % self.options["settings"],
)
def create_project(self):
"""
Django 1.4 project structure
+ project-root-folder/
- manage.py (skipped)
+ project
- __init__.py
- settings.py
- urls.py
- wsgi.py (skipped)
Django duke final project structure
+ project-root-folder/
- setup.py
- bootstrap.py
- buildout.cfg
+ project
- __init__.py
- settings.py
- local_settings.py
- urls.py
+ conf/
- default.py
- dev.py
"""
base_dir = self.buildout["buildout"]["directory"]
project_dir = os.path.join(base_dir, self.options["project"])
bin_path = os.path.join(base_dir, ".duke/bin/")
tmp_path = os.path.join("/tmp/", self.options["project"])
if os.path.exists(project_dir):
logger.info("Skipping creating of project: %(project)s " "since it exists" % self.options)
else:
# p = "',\n '".join(self.buildout['python']['extra-paths'].split('\n'))
logger.info("Creating project: %s " % self.options["project"])
logger.info(self.options["extra-paths"])
if self.options.get("template"):
options = "--template=%s --extension=py,rst,md,default" % self.options["template"]
else:
options = ""
# Lazy fix for weird file permission problem..
self.command("chmod a+x " + os.path.join(bin_path, "django"))
django_admin = None
for d in os.listdir(self.buildout["buildout"]["eggs-directory"]):
p = os.path.join(self.buildout["buildout"]["eggs-directory"], d)
if d.startswith("Django") and os.path.exists(os.path.join(p, "django/bin/django-admin.py")):
django_admin = os.path.join(p, "django/bin/django-admin.py")
self.command(
"%(django)s startproject %(options)s %(project)s %(dest)s"
% {
"django": django_admin or os.path.join(bin_path, "django"),
"project": self.options["project"],
"options": options,
"dest": base_dir,
}
)
# def make_scripts(self, extra_paths, ws):
# # The scripts function uses a script_template variable hardcoded
# # in Buildout to generate the script file. Since the wsgi file
# # needs to create a callable application function rather than call
# # a script, monkeypatch the script template here.
# _script_template = zc.buildout.easy_install.script_template
# zc.buildout.easy_install.script_template = \
# zc.buildout.easy_install.script_header + WSGI_TEMPLATE
# generated = zc.buildout.easy_install.scripts(
# [('%s.wsgi' % self.options['script-name'],
# 'djangodukerecipe.wsgi', 'main')],
# ws, self.options['executable'],
# self.options['bin-directory'], extra_paths = extra_paths,
# #arguments= "'%s.%s'" % (self.options["project"],
# # self.options['settings'])
# arguments = "'%s'" % self.options.get("settings", self.options.get('project') + ".settings")
# )
# zc.buildout.easy_install.script_template = _script_template
# return generated
def update(self):
self.install()
def command(self, cmd, **kwargs):
logger.info("Executing %s " % cmd)
output = subprocess.PIPE
if self.buildout["buildout"].get("verbosity"):
output = None
command = subprocess.Popen(cmd, shell=True, stdout=output, **kwargs)
return command.wait()
def setup_working_set(self):
egg = Egg(self.buildout, 'Django', self.options)
self.working_set = egg.working_set(self.eggs)
def setup_working_set(self):
egg = Egg(
self.buildout, 'Django', self.options
)
self.working_set = egg.working_set(self.eggs)
class Recipe(object):
def __init__(self, buildout, name, options):
self.buildout, self.options, self.name = buildout, options, name
self.egg = Egg(buildout, options['recipe'], options)
bin_test = os.path.join(buildout['buildout']['bin-directory'], 'test')
self.testscript = self.options.get('test-script', bin_test)
if 'eggs' not in self.options:
# Fallback to [test]'s eggs if available.
if 'test' in self.buildout:
self.options['eggs'] = self.buildout['test'].get('eggs')
self.default_policy = self.options.get('default-policy', 'include').strip()
self.exclude = self.options.get('exclude', '').split()
self.exclude_groups = self.options.get('exclude-groups', '').split()
self.include = self.options.get('include', '').split()
self.include_groups = self.options.get('include-groups', '').split()
options['location'] = os.path.join(
buildout['buildout']['parts-directory'],
self.name,
)
def install(self):
options = self.options
location = options['location']
reqs, ws = self.egg.working_set(['plone.recipe.alltests'])
packages = []
paths = {}
for dist in ws.by_key.values():
name = dist.project_name
packages.append(name)
paths[name] = dist.location
excludes = [re.compile(e) for e in self.exclude]
includes = [re.compile(e) for e in self.include]
filtered_packages = []
for p in list(set(packages) - EXCLUDE_PACKAGES):
match = False
if self.default_policy == 'include':
for e in excludes:
if e.search(p) is not None:
match = True
break
if not match:
filtered_packages.append(p)
elif self.default_policy == 'exclude':
for i in includes:
if i.search(p) is not None:
match = True
break
if match:
filtered_packages.append(p)
packages = filtered_packages
# Allow to map distribution names to different package names
pmap = dict()
package_map = options.get('package-map', '').strip()
if package_map:
pmap = self.buildout[package_map]
for k, v in pmap.items():
if k in packages:
packages.remove(k)
packages.append(v)
paths[v] = paths[k]
del paths[k]
packages.sort()
# Allow to group multiple packages into one test run
groups = dict()
groups_section = options.get('groups', '').strip()
if groups_section:
data = self.buildout[groups_section]
for k, v in data.items():
if self.default_policy == 'include':
if k in self.exclude_groups:
for p in v.split():
if p in packages:
packages.remove(p)
else:
groups[k] = v.split()
elif self.default_policy == 'exclude' and k in self.include_groups:
groups[k] = v.split()
easy_install.scripts(
[(self.name, 'plone.recipe.alltests.runner', 'main')],
ws, options['executable'], options['bin-directory'],
arguments=dict(
packages=packages,
testscript=self.testscript,
paths=paths,
groups=groups,
),
)
return location
def update(self):
return self.install()