def dispatch(self, packer_conf, root_folder, global_conf=None):
carrier = Carrier(pecan.conf.rabbit_server,
pecan.conf.rabbit_port,
pecan.conf.rabbit_user,
pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost,
pecan.conf.rabbit_db
)
# carrier.declare_queue('docker.queue')
for distro_name, build_conf in packer_conf.items():
if not hasattr(self.build, "forced_distro") or (
self.build.forced_distro == distro_name or
not self.build.forced_distro):
if 'type' not in build_conf:
# Default type is docker to keep backward compat
build_conf['type'] = 'docker'
# if not build_conf['type'] == 'docker':
queue = "%s.queue" % build_conf['type']
carrier.declare_queue(queue)
# Add global external repos
if not build_conf.get('repos'):
build_conf['repos'] = {}
if not build_conf.get('repos').get('rpm'):
build_conf['repos']['rpm'] = []
if global_conf:
build_conf['repos']['rpm'] += global_conf['repos']['rpm']
#
build_conf['distro'] = distro_name
build_conf['branch'] = self.branch
build_conf['root_folder'] = root_folder
job_data = {
'distro': distro_name,
'username': self.build.username,
'project_name': self.build.project_name,
'build_id': self.build.id_,
}
job = Job(job_data)
job.create()
message = {
'distro_name': distro_name,
'build_conf': build_conf,
'root_folder': root_folder,
'log_path': get_logger_path(self.build),
'id_': self.id_,
'job_id': job.id_,
'build': self.build.dumps(),
'build_path': self.build.get_folder_path()
}
if not carrier.send_message(message, queue):
self.logger.error("Can't post message to rabbitmq")
else:
self.logger.info("Posted build to %s" % queue)
self.build.inc_job_count()
def dispatch(self, packer_conf, root_folder, global_conf=None):
carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
pecan.conf.rabbit_user, pecan.conf.rabbit_password,
pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
# carrier.declare_queue('docker.queue')
for distro_name, build_conf in packer_conf.items():
if not hasattr(self.build, "forced_distro") or (
self.build.forced_distro == distro_name
or not self.build.forced_distro):
if 'type' not in build_conf:
# Default type is docker to keep backward compat
build_conf['type'] = 'docker'
# if not build_conf['type'] == 'docker':
queue = "%s.queue" % build_conf['type']
carrier.declare_queue(queue)
# Add global external repos
if not build_conf.get('repos'):
build_conf['repos'] = {}
if not build_conf.get('repos').get('rpm'):
build_conf['repos']['rpm'] = []
if global_conf:
build_conf['repos']['rpm'] += global_conf['repos']['rpm']
#
build_conf['distro'] = distro_name
build_conf['branch'] = self.branch
build_conf['root_folder'] = root_folder
job_data = {
'distro': distro_name,
'username': self.build.username,
'project_name': self.build.project_name,
'build_id': self.build.id_,
}
job = Job(job_data)
job.create()
message = {
'distro_name': distro_name,
'build_conf': build_conf,
'root_folder': root_folder,
'log_path': get_logger_path(self.build),
'id_': self.id_,
'job_id': job.id_,
'build': self.build.dumps(),
'build_path': self.build.get_folder_path()
}
if not carrier.send_message(message, queue):
self.logger.error("Can't post message to rabbitmq")
else:
self.logger.info("Posted build to %s" % queue)
self.build.inc_job_count()
class OsxPacker(object):
def __init__(self, builder, config):
self.config = config
self.builder = builder
self.distro = "osx"
self.source_url = builder.source_url
self.source_type = builder.source_type
self.branch = builder.build.branch
self.folder = builder.folder
self.config['version'] = 'nightly'
self.config['release'] = '1'
self.config['name'] = 'ring'
job_data = {
'distro': self.distro,
'username': self.builder.build.username,
'project_name': self.builder.build.project_name,
'build_id': self.builder.build.id_,
}
self.job = Job(job_data)
self.job.create()
self.folder_output = self.job.get_folder_output()
self.job_tmp_folder = self.job.get_folder_tmp()
if not os.path.exists(self.folder_output):
os.makedirs(self.folder_output)
if not os.path.exists(self.job_tmp_folder):
os.makedirs(self.job_tmp_folder)
self.logger = get_logger_job(self.job)
def set_status(self, status):
self.job.set_status(status)
def set_build_time(self, build_time):
self.job.set_build_time(build_time)
def run(self):
steps = (
('cloning', self.clone),
('reading_conf', self.reading_conf),
('setup', self.setup),
('compiling', self.compile_),
('transfering', self.transfert_output),
)
for step_name, step_function in steps:
self.set_status(step_name)
if step_function() is not True:
self.logger.debug("Task failed during step: %s", step_name)
self.set_status('failed')
return False
# Save package name in build.cfg
if (self.config.get('name') is not None
and self.builder.build.package_name is None):
self.builder.build.package_name = self.config.get('name')
self.builder.build._save()
self.set_status('succeeded')
return True
def clone(self):
self.logger.info("Cloning main repo")
self.logger.info(self.job.get_folder_tmp())
cmds = [
"cd %s" % self.job.get_folder_tmp(),
"git clone -b %s %s source/" % (self.branch, self.source_url),
]
command = " && "
command = command.join(cmds)
return self.exec_cmd(command)
def reading_conf(self):
self.logger.info("Reading conf from main repo")
conf_file = "%s/source/.packer.yml" % self.job.get_folder_tmp()
try:
stream = open(conf_file, "r")
except IOError:
self.logger.error(".packer.yml not present")
return False
docs = yaml.load_all(stream)
osx_conf = {}
for doc in docs:
for key, value in doc.items():
osx_conf[key] = value
try:
self.dependencies = osx_conf['osx']['brew_deps']
self.commands = osx_conf['osx']['commands']
except KeyError:
self.logger.error("Malformed .packer.yml file")
return False
return True
def setup(self):
# Installing dependencies
for depen in self.dependencies:
cmd_list = ["brew", "install"]
cmd_list.extend(depen.split(" "))
self.logger.info("Installing dependency: %s" % depen)
process = subprocess.Popen(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
self.logger.debug(stdout)
self.logger.info(stderr)
if process.returncode:
self.logger.error("Error in setup: %d" % process.returncode)
return False
return True
def compile_(self):
self.logger.info("Start compiling")
# Compiling ring-daemon
cd_command = ["cd %s" % self.job.get_folder_tmp()]
self.commands = cd_command + self.commands
long_command = " && "
long_command = long_command.join(self.commands)
long_command = long_command % {
"prefix_path": pecan.conf.workspace_path
}
self.logger.info("Compiling")
process = subprocess.Popen(long_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stdout, stderr = process.communicate()
self.logger.debug(stdout)
self.logger.info(stderr)
if process.returncode:
self.logger.error("Error in setup: %d" % process.returncode)
return False
return True
def transfert_output(self):
self.logger.info("Start package transfert")
# move dmg
try:
origin = (self.job.get_folder_tmp() +
"/libringclient/ring-client-macosx/build/Ring.dmg")
destination = (self.builder.build.get_folder_path() +
"/output/osx/Ring.dmg")
os.rename(origin, destination)
except Exception:
self.logger.error("Can't move .dmg file")
return False
# Delete useless files
try:
shutil.rmtree(self.job.get_folder_path() + "/tmp")
except Exception as e:
self.logger.error("Couldn't remove tmp job files: " + e)
host = pecan.conf.origin_host
user = pecan.conf.origin_user
key = pecan.conf.origin_key
# TODO: Correct source and dest (package_dir and path), output/*
# TODO: Add the transfert of jobs/*
path = self.builder.origin_build_path
package_dir = self.builder.build.get_folder_path() + "/*"
# transfert_command = "scp -r -i %s %s %s@%s:%s" % (
transfert_command = 'rsync -az -e "ssh -i %s" %s %s@%s:%s --exclude jobs/*/tmp' % (
key, package_dir, user, host, path)
self.logger.info(transfert_command)
command_res = self.exec_cmd(transfert_command)
self.logger.info(command_res)
return command_res
def clean(self):
try:
shutil.rmtree(self.builder.build.get_folder_path())
except Exception:
self.logger.error("Could not remove temps files: %s" %
(self.builder.build.get_folder_path()))
return False
return True
def exec_cmd(self, cmds):
process = subprocess.Popen(cmds,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stdout, stderr = process.communicate()
self.logger.debug(stdout)
self.logger.info(stderr)
if process.returncode:
self.logger.error("Error in setup: %d" % process.returncode)
return False
return True
class Packer(object):
def __init__(self, builder, config):
self.config = config
self.builder = builder
self.distro = supported_distros[config['distro']]
self.source_url = builder.source_url
self.source_type = builder.source_type
self.cli = builder.cli
self.folder = builder.folder
job_data = {
'distro': config['distro'],
'username': self.builder.build.username,
'project_name': self.builder.build.project_name,
'build_id': self.builder.build.id_,
}
self.job = Job(job_data)
self.job.create()
self.folder_output = self.job.get_folder_output()
self.job_tmp_folder = self.job.get_folder_tmp()
if not os.path.exists(self.folder_output):
os.makedirs(self.folder_output)
if not os.path.exists(self.job_tmp_folder):
os.makedirs(self.job_tmp_folder)
self.logger = get_logger_job(self.job)
self.container_tag = "joulupukki:" + config['distro'].replace(":", "_")
self.container = None
def set_status(self, status):
self.job.set_status(status)
def set_build_time(self, build_time):
self.job.set_build_time(build_time)
def run(self):
steps = (('setup', self.setup),
('preparing', self.parse_specdeb),
('save_data', self.save_data),
('building', self.docker_build),
('packaging', self.docker_run),
('finishing', self.get_output),
('cleaning', self.clean_up),
)
for step_name, step_function in steps:
self.set_status(step_name)
if step_function() is not True:
self.logger.debug("Task failed during step: %s", step_name)
self.set_status('failed')
return False
self.set_status('succeeded')
return True
def save_data(self):
if self.config.get('name') is not None and self.builder.build.package_name is None:
self.builder.build.package_name = self.config.get('name')
if self.config.get('version') is not None and self.builder.build.package_version is None:
self.builder.build.package_version = self.config.get('version')
if self.config.get('release') is not None and self.builder.build.package_release is None:
self.builder.build.package_release = self.config.get('release')
self.builder.build._save()
return True
def parse_specdeb(self):
return False
def docker_build(self):
return False
def docker_run(self):
return False
def get_output(self):
return False
def setup(self):
# Remove images
for image in self.cli.images():
if not any([True for tag in image['RepoTags'] if tag.startswith("joulupukki:")]):
continue
try:
creation_date = datetime.fromtimestamp(image.get('Created'))
delta_time = datetime.now() - creation_date
if delta_time.days >= 1:
self.logger.debug('Deleting docker image: %s', image['Id'])
self.cli.remove_image(image['Id'])
except Exception as error:
self.logger.debug('Cannot deleting docker image: %s'
' - Error: %s', image['Id'], error)
return True
def clean_up(self):
# Set end time
self.job.set_end_time(time.time())
# Delete container
self.logger.debug('Deleting docker container: %s', self.container['Id'])
self.cli.remove_container(self.container['Id'])
if os.path.isdir(self.job_tmp_folder):
shutil.rmtree(self.job_tmp_folder)
return True
class Packer(object):
def __init__(self, builder, config):
self.config = config
self.builder = builder
self.distro = supported_distros[config['distro']]
self.source_url = builder.source_url
self.source_type = builder.source_type
self.cli = builder.cli
self.folder = builder.folder
job_data = {
'distro': config['distro'],
'username': self.builder.build.username,
'project_name': self.builder.build.project_name,
'build_id': self.builder.build.id_,
}
self.job = Job(job_data)
self.job.create()
self.folder_output = self.job.get_folder_output()
self.job_tmp_folder = self.job.get_folder_tmp()
if not os.path.exists(self.folder_output):
os.makedirs(self.folder_output)
if not os.path.exists(self.job_tmp_folder):
os.makedirs(self.job_tmp_folder)
self.logger = get_logger_job(self.job)
self.container_tag = "joulupukki:" + config['distro'].replace(":", "_")
self.container = None
def set_status(self, status):
self.job.set_status(status)
def set_build_time(self, build_time):
self.job.set_build_time(build_time)
def run(self):
steps = (
('setup', self.setup),
('preparing', self.parse_specdeb),
('save_data', self.save_data),
('building', self.docker_build),
('packaging', self.docker_run),
('finishing', self.get_output),
('cleaning', self.clean_up),
)
for step_name, step_function in steps:
self.set_status(step_name)
if step_function() is not True:
self.logger.debug("Task failed during step: %s", step_name)
self.set_status('failed')
return False
self.set_status('succeeded')
return True
def save_data(self):
if self.config.get(
'name'
) is not None and self.builder.build.package_name is None:
self.builder.build.package_name = self.config.get('name')
if self.config.get(
'version'
) is not None and self.builder.build.package_version is None:
self.builder.build.package_version = self.config.get('version')
if self.config.get(
'release'
) is not None and self.builder.build.package_release is None:
self.builder.build.package_release = self.config.get('release')
self.builder.build._save()
return True
def parse_specdeb(self):
return False
def docker_build(self):
return False
def docker_run(self):
return False
def get_output(self):
return False
def setup(self):
# Remove images
for image in self.cli.images():
if not any([
True for tag in image['RepoTags']
if tag.startswith("joulupukki:")
]):
continue
try:
creation_date = datetime.fromtimestamp(image.get('Created'))
delta_time = datetime.now() - creation_date
if delta_time.days >= 1:
self.logger.debug('Deleting docker image: %s', image['Id'])
self.cli.remove_image(image['Id'])
except Exception as error:
self.logger.debug(
'Cannot deleting docker image: %s'
' - Error: %s', image['Id'], error)
return True
def clean_up(self):
# Set end time
self.job.set_end_time(time.time())
# Delete container
self.logger.debug('Deleting docker container: %s',
self.container['Id'])
self.cli.remove_container(self.container['Id'])
if os.path.isdir(self.job_tmp_folder):
shutil.rmtree(self.job_tmp_folder)
return True
class OsxPacker(object):
def __init__(self, builder, config):
self.config = config
self.builder = builder
self.distro = "osx"
self.source_url = builder.source_url
self.source_type = builder.source_type
self.branch = builder.build.branch
self.folder = builder.folder
self.config['version'] = 'nightly'
self.config['release'] = '1'
self.config['name'] = 'ring'
job_data = {
'distro': self.distro,
'username': self.builder.build.username,
'project_name': self.builder.build.project_name,
'build_id': self.builder.build.id_,
}
self.job = Job(job_data)
self.job.create()
self.folder_output = self.job.get_folder_output()
self.job_tmp_folder = self.job.get_folder_tmp()
if not os.path.exists(self.folder_output):
os.makedirs(self.folder_output)
if not os.path.exists(self.job_tmp_folder):
os.makedirs(self.job_tmp_folder)
self.logger = get_logger_job(self.job)
def set_status(self, status):
self.job.set_status(status)
def set_build_time(self, build_time):
self.job.set_build_time(build_time)
def run(self):
steps = (
('cloning', self.clone),
('reading_conf', self.reading_conf),
('setup', self.setup),
('compiling', self.compile_),
('transfering', self.transfert_output),
)
for step_name, step_function in steps:
self.set_status(step_name)
if step_function() is not True:
self.logger.debug("Task failed during step: %s", step_name)
self.set_status('failed')
return False
# Save package name in build.cfg
if (self.config.get('name') is not None and
self.builder.build.package_name is None):
self.builder.build.package_name = self.config.get('name')
self.builder.build._save()
self.set_status('succeeded')
return True
def clone(self):
self.logger.info("Cloning main repo")
self.logger.info(self.job.get_folder_tmp())
cmds = [
"cd %s" % self.job.get_folder_tmp(),
"git clone -b %s %s source/" % (self.branch, self.source_url),
]
command = " && "
command = command.join(cmds)
return self.exec_cmd(command)
def reading_conf(self):
self.logger.info("Reading conf from main repo")
conf_file = "%s/source/.packer.yml" % self.job.get_folder_tmp()
try:
stream = open(conf_file, "r")
except IOError:
self.logger.error(".packer.yml not present")
return False
docs = yaml.load_all(stream)
osx_conf = {}
for doc in docs:
for key, value in doc.items():
osx_conf[key] = value
try:
self.dependencies = osx_conf['osx']['brew_deps']
self.commands = osx_conf['osx']['commands']
except KeyError:
self.logger.error("Malformed .packer.yml file")
return False
return True
def setup(self):
# Installing dependencies
for depen in self.dependencies:
cmd_list = ["brew", "install"]
cmd_list.extend(depen.split(" "))
self.logger.info("Installing dependency: %s" % depen)
process = subprocess.Popen(
cmd_list,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
self.logger.debug(stdout)
self.logger.info(stderr)
if process.returncode:
self.logger.error("Error in setup: %d" % process.returncode)
return False
return True
def compile_(self):
self.logger.info("Start compiling")
# Compiling ring-daemon
cd_command = ["cd %s" % self.job.get_folder_tmp()]
self.commands = cd_command + self.commands
long_command = " && "
long_command = long_command.join(self.commands)
long_command = long_command % {
"prefix_path": pecan.conf.workspace_path
}
self.logger.info("Compiling")
process = subprocess.Popen(
long_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True
)
stdout, stderr = process.communicate()
self.logger.debug(stdout)
self.logger.info(stderr)
if process.returncode:
self.logger.error("Error in setup: %d" % process.returncode)
return False
return True
def transfert_output(self):
self.logger.info("Start package transfert")
# move dmg
try:
origin = (self.job.get_folder_tmp() +
"/libringclient/ring-client-macosx/build/Ring.dmg")
destination = (self.builder.build.get_folder_path() +
"/output/osx/Ring.dmg")
os.rename(origin, destination)
except Exception:
self.logger.error("Can't move .dmg file")
return False
# Delete useless files
try:
shutil.rmtree(self.job.get_folder_path() + "/tmp")
except Exception as e:
self.logger.error("Couldn't remove tmp job files: " + e)
host = pecan.conf.origin_host
user = pecan.conf.origin_user
key = pecan.conf.origin_key
# TODO: Correct source and dest (package_dir and path), output/*
# TODO: Add the transfert of jobs/*
path = self.builder.origin_build_path
package_dir = self.builder.build.get_folder_path() + "/*"
# transfert_command = "scp -r -i %s %s %s@%s:%s" % (
transfert_command = 'rsync -az -e "ssh -i %s" %s %s@%s:%s --exclude jobs/*/tmp' % (
key,
package_dir,
user,
host,
path
)
self.logger.info(transfert_command)
command_res = self.exec_cmd(transfert_command)
self.logger.info(command_res)
return command_res
def clean(self):
try:
shutil.rmtree(self.builder.build.get_folder_path())
except Exception:
self.logger.error("Could not remove temps files: %s" % (
self.builder.build.get_folder_path()
))
return False
return True
def exec_cmd(self, cmds):
process = subprocess.Popen(
cmds,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True
)
stdout, stderr = process.communicate()
self.logger.debug(stdout)
self.logger.info(stderr)
if process.returncode:
self.logger.error("Error in setup: %d" % process.returncode)
return False
return True