def __init__(self, project):
self.project = Projects(project)
self.templates_dir = BrickConfig().get('workspace', 'template_dir')
self.git = git.Git(self.project)
self.build_system = BrickConfig().get('build', 'system')
self.build_options = BuildOptions(self.git.workdir)
self.build_container = None
self.workspace = "%s/%s" % (
BrickConfig().get('workspace', 'dir'),
self.project.name,
)
self.real_workspace = "%s/%s" % (BrickConfig().get(
'workspace', 'dir'), self.project.name)
if self.build_system == 'rpm':
self.mod_install_cmd = self.project.install_cmd.replace(
'BUILDROOT', '%{buildroot}')
elif self.build_system == 'deb' or self.build_system == None:
self.mod_install_cmd = self.project.install_cmd.replace(
'BUILDROOT', 'debian/tmp')
if not os.path.isdir(self.workspace):
os.makedirs(self.workspace)
if not os.path.isdir(os.path.join(self.workspace, 'log')):
os.makedirs(os.path.join(self.workspace, 'log'))
self.stdout = None
self.stderr = self.stdout
def local_repo(self, distribution, files):
archive_conf_data = """Dir {
ArchiveDir "%s/%s";
};
BinDirectory "dists/unstable" {
Packages "dists/unstable/main/binary-amd64/Packages";
SrcPackages "dists/unstable/main/source/Sources";
};
BinDirectory "dists/stable" {
Packages "dists/stable/main/binary-amd64/Packages";
SrcPackages "dists/stable/main/source/Sources";
};
BinDirectory "dists/testing" {
Packages "dists/testing/main/binary-amd64/Packages";
SrcPackages "dists/testing/main/source/Sources";
};
BinDirectory "dists/experimental" {
Packages "dists/experimental/main/binary-amd64/Packages";
SrcPackages "dists/experimental/main/source/Sources";
};"""
repo_bin_path = os.path.join(BrickConfig().get("local_repo", "dir"),
self.project.group_name, 'dists/%s/main/binary-amd64/' % distribution)
repo_src_path = os.path.join(BrickConfig().get("local_repo", "dir"),
self.project.group_name, 'dists/%s/main/source/' % distribution)
archive_conf_file = os.path.join(
BrickConfig().get("local_repo", "dir"),
self.project.group_name,
'archive.conf')
if not os.path.isdir(repo_bin_path) or not os.path.isdir(repo_src_path):
os.makedirs(repo_bin_path)
os.makedirs(repo_src_path)
if not os.path.isfile(archive_conf_file):
open(archive_conf_file, 'w').write(
archive_conf_data % (
BrickConfig().get("local_repo", "dir"),
self.project.group_name
))
os.chdir(self.builder.workspace)
for f in files:
if f.endswith('.dsc') or f.endswith('.tar.gz'):
print repo_src_path, f
shutil.copy(f, os.path.join(repo_src_path, f))
elif f.endswith('.deb'):
print repo_bin_path, f
shutil.copy(f, os.path.join(repo_bin_path, f))
repo_base_path = os.path.join(BrickConfig().get('local_repo', 'dir'), self.project.group_name)
def __init__(self, project, workdir=None):
_workdir = workdir
if not _workdir:
_workdir = BrickConfig().get('workspace', 'dir')
self.workdir = os.path.join(_workdir, project.name)
self.project = project
def upload_files(self, distribution, files):
repository_url, user, passwd = self.project.repository()
if not repository_url:
return 0
os.chdir(BrickConfig().get('workspace', 'dir'))
workspace = BrickConfig().get('workspace', 'dir')
ftp = ftplib.FTP(repository_url, user, passwd)
try:
ftp.cwd(distribution)
for f in files:
log.info("\t%s: " % os.path.join(workspace, f))
ftp.storbinary("STOR %s" % f,
open(os.path.join(workspace, f), 'rb'))
log.info("done.")
except Exception, e:
log.info(repr(e))
def send_job(self, project_name, branch, release, version):
log.msg('sched build: %s [%s:%s]' % (project_name, release, version))
brickconfig = BrickConfig()
builder.build_project({
'project': project_name,
'branch': branch,
'release': release,
'version': version,
})
def _exec_git(self, cmd=[], cwd=None, stdout=None):
if stdout is None:
stdout = devnull
if (cwd is None):
cwd = BrickConfig().get('workspace', 'dir')
return subprocess.Popen(" ".join(cmd),
shell=True,
stdout=stdout,
stderr=stdout,
close_fds=True,
cwd=cwd)
def upload(self, branch):
glob_str = '%s/%s_%s_*.changes' % (BrickConfig().get(
'workspace',
'dir'), self.project.name, self.project.version(branch))
changes_file = glob.glob(glob_str)
log.info(changes_file)
distribution, files = self.parse_changes(changes_file[0])
try:
self.upload_files(distribution, files)
upload_file = changes_file[0].replace('.changes', '.upload')
with open(upload_file, 'w') as tmpfh:
tmpfh.write("done")
except Exception, e:
log.error("Package could not be uploaded: %s", e)
from utils import pystache
from twisted.internet import threads, reactor, defer
from config import BrickConfig
from model.project import Project
# from builder_rpm import BuilderRpm
from builder_deb import BuilderDeb
from build_options import BuildOptions
from build_container import BuildContainer
from current_build import CurrentBuild
#from dreque import Dreque
config = BrickConfig()
redis_server = config.get('redis', 'redis-server')
log_file = config.get('log', 'file')
#queue = Dreque(redis_server)
logging.basicConfig(filename=log_file, level=logging.DEBUG)
log = logging.getLogger('builder')
@defer.inlineCallbacks
def build_project(kargs):
builder = Builder(kargs['project'])
kargs.pop('project')
yield builder.build_project(**kargs)
def build(self,
branch,
last_tag=None,
force_version=None,
force_release=None):
templates = {}
templates_dir = os.path.join(self.builder.templates_dir, 'deb')
control_data_new = None
self.build_info = BuildInfo(self.project.name)
logfile = os.path.join(
BrickConfig().get("workspace", "dir"), 'log',
'%s.%s.log' % (self.project.name, self.build_info.build_id))
log.info("build log file: %s" % logfile)
self.build_info.log(logfile)
self.stdout = open(logfile, 'a+')
self.stderr = self.stdout
self.debian_dir = os.path.join(self.builder.workdir, 'debian')
if last_tag is not None:
os.environ.update({'BRICKLAYER_RELEASE': last_tag.split('_')[0]})
os.environ.update({'BRICKLAYER_TAG': last_tag})
# Not now
#self.build_install_deps()
if self.project.install_prefix is None:
self.project.install_prefix = 'opt'
if not self.project.install_cmd:
self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % (
self.project.install_prefix)
changelog_entry, changelog_data = self.configure_changelog(branch)
if changelog_entry and changelog_data:
if last_tag != None and last_tag.startswith('stable'):
self.project.version('stable', last_tag.split('_')[1])
changelog_data.update({
'version': self.project.version('stable'),
'branch': 'stable'
})
self.build_info.version(self.project.version('stable'))
self.build_info.release('stable')
elif last_tag != None and last_tag.startswith('testing'):
self.project.version('testing', last_tag.split('_')[1])
changelog_data.update({
'version':
self.project.version('testing'),
'branch':
'testing'
})
self.build_info.version(self.project.version('testing'))
self.build_info.release('testing')
elif last_tag != None and last_tag.startswith('unstable'):
self.project.version('unstable', last_tag.split('_')[1])
changelog_data.update({
'version':
self.project.version('unstable'),
'branch':
'unstable'
})
self.build_info.version(self.project.version('unstable'))
self.build_info.release('unstable')
else:
"""
otherwise it should change the distribution to experimental
"""
version_list = self.project.version(branch).split('.')
version_list[len(version_list) - 1] = str(
int(version_list[len(version_list) - 1]) + 1)
self.project.version(branch, '.'.join(version_list))
changelog_data.update({
'version': self.project.version(branch),
'branch': 'experimental'
})
self.build_info.version(self.project.version(branch))
self.build_info.release('experimental:%s' % branch)
with file(
os.path.join(self.builder.workdir, 'debian', 'changelog'),
'w') as fh:
fh.write(changelog_entry % changelog_data)
else:
self.build_info.version(force_version)
self.build_info.release(force_release)
self.project.version(branch, force_version)
self.project.version(force_release, force_version)
rvm_env = {}
rvm_rc = os.path.join(self.builder.workdir, '.rvmrc')
rvm_rc_example = rvm_rc + ".example"
has_rvm = False
if os.path.isfile(rvm_rc):
has_rvm = True
elif os.path.isfile(rvm_rc_example):
has_rvm = True
rvm_rc = rvm_rc_example
if has_rvm:
with open(rvm_rc) as tmpfh:
rvmexec = tmpfh.read()
log.info("RVMRC: %s" % rvmexec)
# I need the output not to log on file
rvm_cmd = subprocess.Popen('/usr/local/rvm/bin/rvm info %s' %
rvmexec.split()[1],
shell=True,
stdout=subprocess.PIPE)
rvm_cmd.wait()
for line in rvm_cmd.stdout.readlines():
if 'PATH' in line or 'HOME' in line:
name, value = line.split()
rvm_env[name.strip(':')] = value.strip('"')
rvm_env['HOME'] = os.environ['HOME']
if len(rvm_env.keys()) < 1:
rvm_env = os.environ
else:
os_env = dict(os.environ)
for k in ("PATH", "GEM_HOME", "BUNDLER_PATH"):
if (k in os_env):
del (os_env[k])
rvm_env.update(os_env)
log.info(rvm_env)
os.chmod(os.path.join(self.debian_dir, 'rules'),
stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH)
dpkg_cmd = self.builder._exec([
'dpkg-buildpackage', '-rfakeroot', '-tc',
'-k%s' % BrickConfig().get('gpg', 'keyid')
],
cwd=self.builder.workdir,
env=rvm_env,
stdout=self.stdout,
stderr=self.stderr,
close_fds=True)
dpkg_cmd.wait()
clean_cmd = self.builder._exec(['dh', 'clean'],
cwd=self.builder.workdir,
stdout=self.stdout,
stderr=self.stderr,
close_fds=True)
clean_cmd.wait()
from config import BrickConfig
print BrickConfig('./config/bricklayer.ini').get('databases', 'uri')
class BuilderDeb():
def __init__(self, builder):
self.builder = builder
self.project = self.builder.project
log.info("Debian builder initialized: %s" % self.builder.workdir)
log.info("Building with options: %s" % self.builder.build_options.options)
def build(self, branch, last_tag=None, force_version=None, force_release=None):
templates = {}
templates_dir = os.path.join(self.builder.templates_dir, 'deb')
debian_dir = os.path.join(self.builder.workdir, 'debian')
control_data_new = None
self.build_info = BuildInfo(self.project.name)
logfile = os.path.join(
self.builder.workspace, 'log', '%s.%s.log' % (
self.project.name, self.build_info.build_id
)
)
self.build_info.log(logfile)
self.stdout = open(logfile, 'a+')
self.stderr = self.stdout
if self.project.install_prefix is None:
self.project.install_prefix = 'opt'
if not self.project.install_cmd :
self.project.install_cmd = 'cp -r \`ls | grep -v debian\` debian/tmp/%s' % (
self.project.install_prefix
)
template_data = {
'name': self.project.name,
'version': "%s" % (self.project.version(branch)),
'build_cmd': self.project.build_cmd,
'install_cmd': self.builder.mod_install_cmd,
'username': self.project.username,
'email': self.project.email,
'date': time.strftime("%a, %d %h %Y %T %z"),
}
changelog = os.path.join(self.builder.workdir, 'debian', 'changelog')
if hasattr(self.builder.build_options, 'changelog') and self.builder.build_options.changelog:
if os.path.isfile(changelog):
os.rename(changelog, "%s.save" % changelog)
def read_file_data(f):
template_fd = open(os.path.join(templates_dir, f))
templates[f] = pystache.template.Template(template_fd.read()).render(context=template_data)
template_fd.close()
if not os.path.isdir(debian_dir):
map(read_file_data, ['changelog', 'control', 'rules'])
os.makedirs( os.path.join( debian_dir, self.project.name, self.project.install_prefix))
for filename, data in templates.iteritems():
open(os.path.join(debian_dir, filename), 'w').write(data)
changelog_entry = """%(name)s (%(version)s) %(branch)s; urgency=low
* Latest commits
%(commits)s
-- %(username)s <%(email)s> %(date)s
"""
changelog_data = {
'name': self.project.name,
'version': self.project.version(branch),
'branch': branch,
'commits': ' '.join(self.builder.git.log()),
'username': self.project.username,
'email': self.project.email,
'date': time.strftime("%a, %d %h %Y %T %z"),
}
if last_tag != None and last_tag.startswith('stable'):
self.project.version('stable', last_tag.split('_')[1])
changelog_data.update({'version': self.project.version('stable'), 'branch': 'stable'})
self.build_info.version(self.project.version('stable'))
self.build_info.release('stable')
elif last_tag != None and last_tag.startswith('testing'):
self.project.version('testing', last_tag.split('_')[1])
changelog_data.update({'version': self.project.version('testing'), 'branch': 'testing'})
self.build_info.version(self.project.version('testing'))
self.build_info.release('testing')
elif last_tag != None and last_tag.startswith('unstable'):
self.project.version('unstable', last_tag.split('_')[1])
changelog_data.update({'version': self.project.version('unstable'), 'branch': 'unstable'})
self.build_info.version(self.project.version('unstable'))
self.build_info.release('unstable')
else:
"""
otherwise it should change the distribution to experimental
"""
if self.project.version(branch):
version_list = self.project.version(branch).split('.')
version_list[len(version_list) - 1] = str(int(version_list[len(version_list) - 1]) + 1)
self.project.version(branch, '.'.join(version_list))
changelog_data.update({'version': self.project.version(branch), 'branch': 'experimental'})
self.build_info.version(self.project.version(branch))
self.build_info.release('experimental:%s' % branch)
open(os.path.join(self.builder.workdir, 'debian', 'changelog'), 'w').write(changelog_entry % changelog_data)
else:
self.build_info.version(force_version)
self.build_info.release(force_release)
rvm_env = {}
rvm_rc = os.path.join(self.builder.workdir, '.rvmrc')
rvm_rc_example = rvm_rc + ".example"
has_rvm = False
if os.path.isfile(rvm_rc):
has_rvm = True
elif os.path.isfile(rvm_rc_example):
has_rvm = True
rvm_rc = rvm_rc_example
if has_rvm:
rvmexec = open(rvm_rc).read()
log.info("RVMRC: %s" % rvmexec)
# I need the output not to log on file
rvm_cmd = subprocess.Popen('/usr/local/bin/rvm info %s' % rvmexec.split()[1],
shell=True, stdout=subprocess.PIPE)
rvm_cmd.wait()
for line in rvm_cmd.stdout.readlines():
if 'PATH' in line or 'HOME' in line:
name, value = line.split()
rvm_env[name.strip(':')] = value.strip('"')
rvm_env['HOME'] = os.environ['HOME']
if len(rvm_env.keys()) < 1:
rvm_env = os.environ
else:
try:
os.environ.pop('PATH')
os.environ.pop('GEM_HOME')
os.environ.pop('BUNDLER_PATH')
except Exception, e:
pass
rvm_env.update(os.environ)
os.chmod(os.path.join(debian_dir, 'rules'), stat.S_IRWXU|stat.S_IRWXG|stat.S_IROTH|stat.S_IXOTH)
dpkg_cmd = self.builder._exec(
['dpkg-buildpackage', '-rfakeroot', '-tc', '-k%s' % BrickConfig().get('gpg', 'keyid')],
cwd=self.builder.workdir, env=rvm_env, stdout=self.stdout, stderr=self.stderr
)
dpkg_cmd.wait()
clean_cmd = self.builder._exec(['dh', 'clean'], cwd=self.builder.workdir)
clean_cmd.wait()
if self.builder.build_options.changelog and os.path.isfile("%s.save" % changelog):
os.rename("%s.save" % changelog, changelog)
