Exemplo n.º 1
0
    def dispatch(self, packer_conf, root_folder, global_conf=None):

        carrier = Carrier(pecan.conf.rabbit_server,
                          pecan.conf.rabbit_port,
                          pecan.conf.rabbit_user,
                          pecan.conf.rabbit_password,
                          pecan.conf.rabbit_vhost,
                          pecan.conf.rabbit_db
                          )
        # carrier.declare_queue('docker.queue')
        for distro_name, build_conf in packer_conf.items():
            if not hasattr(self.build, "forced_distro") or (
                    self.build.forced_distro == distro_name or
                    not self.build.forced_distro):
                if 'type' not in build_conf:
                    # Default type is docker to keep backward compat
                    build_conf['type'] = 'docker'

                # if not build_conf['type'] == 'docker':
                queue = "%s.queue" % build_conf['type']
                carrier.declare_queue(queue)

                # Add global external repos
                if not build_conf.get('repos'):
                    build_conf['repos'] = {}
                if not build_conf.get('repos').get('rpm'):
                    build_conf['repos']['rpm'] = []
                if global_conf:
                    build_conf['repos']['rpm'] += global_conf['repos']['rpm']
                #
                build_conf['distro'] = distro_name
                build_conf['branch'] = self.branch
                build_conf['root_folder'] = root_folder
                job_data = {
                    'distro': distro_name,
                    'username': self.build.username,
                    'project_name': self.build.project_name,
                    'build_id': self.build.id_,
                }
                job = Job(job_data)
                job.create()
                message = {
                    'distro_name': distro_name,
                    'build_conf': build_conf,
                    'root_folder': root_folder,
                    'log_path': get_logger_path(self.build),
                    'id_': self.id_,
                    'job_id': job.id_,
                    'build': self.build.dumps(),
                    'build_path': self.build.get_folder_path()
                }
                if not carrier.send_message(message, queue):
                    self.logger.error("Can't post message to rabbitmq")
                else:
                    self.logger.info("Posted build to %s" % queue)
                    self.build.inc_job_count()
Exemplo n.º 2
0
    def dispatch(self, packer_conf, root_folder, global_conf=None):

        carrier = Carrier(pecan.conf.rabbit_server, pecan.conf.rabbit_port,
                          pecan.conf.rabbit_user, pecan.conf.rabbit_password,
                          pecan.conf.rabbit_vhost, pecan.conf.rabbit_db)
        # carrier.declare_queue('docker.queue')
        for distro_name, build_conf in packer_conf.items():
            if not hasattr(self.build, "forced_distro") or (
                    self.build.forced_distro == distro_name
                    or not self.build.forced_distro):
                if 'type' not in build_conf:
                    # Default type is docker to keep backward compat
                    build_conf['type'] = 'docker'

                # if not build_conf['type'] == 'docker':
                queue = "%s.queue" % build_conf['type']
                carrier.declare_queue(queue)

                # Add global external repos
                if not build_conf.get('repos'):
                    build_conf['repos'] = {}
                if not build_conf.get('repos').get('rpm'):
                    build_conf['repos']['rpm'] = []
                if global_conf:
                    build_conf['repos']['rpm'] += global_conf['repos']['rpm']
                #
                build_conf['distro'] = distro_name
                build_conf['branch'] = self.branch
                build_conf['root_folder'] = root_folder
                job_data = {
                    'distro': distro_name,
                    'username': self.build.username,
                    'project_name': self.build.project_name,
                    'build_id': self.build.id_,
                }
                job = Job(job_data)
                job.create()
                message = {
                    'distro_name': distro_name,
                    'build_conf': build_conf,
                    'root_folder': root_folder,
                    'log_path': get_logger_path(self.build),
                    'id_': self.id_,
                    'job_id': job.id_,
                    'build': self.build.dumps(),
                    'build_path': self.build.get_folder_path()
                }
                if not carrier.send_message(message, queue):
                    self.logger.error("Can't post message to rabbitmq")
                else:
                    self.logger.info("Posted build to %s" % queue)
                    self.build.inc_job_count()
Exemplo n.º 3
0
class OsxPacker(object):
    def __init__(self, builder, config):
        self.config = config
        self.builder = builder
        self.distro = "osx"

        self.source_url = builder.source_url
        self.source_type = builder.source_type
        self.branch = builder.build.branch
        self.folder = builder.folder

        self.config['version'] = 'nightly'
        self.config['release'] = '1'
        self.config['name'] = 'ring'

        job_data = {
            'distro': self.distro,
            'username': self.builder.build.username,
            'project_name': self.builder.build.project_name,
            'build_id': self.builder.build.id_,
        }
        self.job = Job(job_data)
        self.job.create()
        self.folder_output = self.job.get_folder_output()

        self.job_tmp_folder = self.job.get_folder_tmp()

        if not os.path.exists(self.folder_output):
            os.makedirs(self.folder_output)
        if not os.path.exists(self.job_tmp_folder):
            os.makedirs(self.job_tmp_folder)

        self.logger = get_logger_job(self.job)

    def set_status(self, status):
        self.job.set_status(status)

    def set_build_time(self, build_time):
        self.job.set_build_time(build_time)

    def run(self):
        steps = (
            ('cloning', self.clone),
            ('reading_conf', self.reading_conf),
            ('setup', self.setup),
            ('compiling', self.compile_),
            ('transfering', self.transfert_output),
        )
        for step_name, step_function in steps:
            self.set_status(step_name)
            if step_function() is not True:
                self.logger.debug("Task failed during step: %s", step_name)
                self.set_status('failed')
                return False
            # Save package name in build.cfg
            if (self.config.get('name') is not None
                    and self.builder.build.package_name is None):
                self.builder.build.package_name = self.config.get('name')
                self.builder.build._save()
        self.set_status('succeeded')
        return True

    def clone(self):
        self.logger.info("Cloning main repo")
        self.logger.info(self.job.get_folder_tmp())
        cmds = [
            "cd %s" % self.job.get_folder_tmp(),
            "git clone -b %s %s source/" % (self.branch, self.source_url),
        ]
        command = " && "
        command = command.join(cmds)

        return self.exec_cmd(command)

    def reading_conf(self):
        self.logger.info("Reading conf from main repo")
        conf_file = "%s/source/.packer.yml" % self.job.get_folder_tmp()
        try:
            stream = open(conf_file, "r")
        except IOError:
            self.logger.error(".packer.yml not present")
            return False
        docs = yaml.load_all(stream)
        osx_conf = {}
        for doc in docs:
            for key, value in doc.items():
                osx_conf[key] = value

        try:
            self.dependencies = osx_conf['osx']['brew_deps']
            self.commands = osx_conf['osx']['commands']
        except KeyError:
            self.logger.error("Malformed .packer.yml file")
            return False
        return True

    def setup(self):
        # Installing dependencies
        for depen in self.dependencies:
            cmd_list = ["brew", "install"]
            cmd_list.extend(depen.split(" "))
            self.logger.info("Installing dependency: %s" % depen)
            process = subprocess.Popen(
                cmd_list,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            stdout, stderr = process.communicate()
            self.logger.debug(stdout)
            self.logger.info(stderr)
            if process.returncode:
                self.logger.error("Error in setup: %d" % process.returncode)
                return False
        return True

    def compile_(self):
        self.logger.info("Start compiling")
        # Compiling ring-daemon
        cd_command = ["cd %s" % self.job.get_folder_tmp()]
        self.commands = cd_command + self.commands
        long_command = " && "
        long_command = long_command.join(self.commands)
        long_command = long_command % {
            "prefix_path": pecan.conf.workspace_path
        }

        self.logger.info("Compiling")
        process = subprocess.Popen(long_command,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=True)
        stdout, stderr = process.communicate()
        self.logger.debug(stdout)
        self.logger.info(stderr)
        if process.returncode:
            self.logger.error("Error in setup: %d" % process.returncode)
            return False
        return True

    def transfert_output(self):
        self.logger.info("Start package transfert")

        # move dmg
        try:
            origin = (self.job.get_folder_tmp() +
                      "/libringclient/ring-client-macosx/build/Ring.dmg")
            destination = (self.builder.build.get_folder_path() +
                           "/output/osx/Ring.dmg")
            os.rename(origin, destination)
        except Exception:
            self.logger.error("Can't move .dmg file")
            return False

        # Delete useless files
        try:
            shutil.rmtree(self.job.get_folder_path() + "/tmp")
        except Exception as e:
            self.logger.error("Couldn't remove tmp job files: " + e)

        host = pecan.conf.origin_host
        user = pecan.conf.origin_user
        key = pecan.conf.origin_key
        # TODO: Correct source and dest (package_dir and path), output/*
        # TODO: Add the transfert of jobs/*
        path = self.builder.origin_build_path
        package_dir = self.builder.build.get_folder_path() + "/*"
        # transfert_command = "scp -r -i %s %s %s@%s:%s" % (
        transfert_command = 'rsync -az -e "ssh -i %s" %s %s@%s:%s --exclude jobs/*/tmp' % (
            key, package_dir, user, host, path)
        self.logger.info(transfert_command)
        command_res = self.exec_cmd(transfert_command)
        self.logger.info(command_res)
        return command_res

    def clean(self):
        try:
            shutil.rmtree(self.builder.build.get_folder_path())
        except Exception:
            self.logger.error("Could not remove temps files: %s" %
                              (self.builder.build.get_folder_path()))
            return False
        return True

    def exec_cmd(self, cmds):
        process = subprocess.Popen(cmds,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   shell=True)
        stdout, stderr = process.communicate()
        self.logger.debug(stdout)
        self.logger.info(stderr)
        if process.returncode:
            self.logger.error("Error in setup: %d" % process.returncode)
            return False
        return True
Exemplo n.º 4
0
class Packer(object):

    def __init__(self, builder, config):

        self.config = config

        self.builder = builder
        self.distro = supported_distros[config['distro']]
        self.source_url = builder.source_url
        self.source_type = builder.source_type
        self.cli = builder.cli
        self.folder = builder.folder

        job_data = {
            'distro': config['distro'],
            'username': self.builder.build.username,
            'project_name': self.builder.build.project_name,
            'build_id': self.builder.build.id_,
        }
        self.job = Job(job_data)
        self.job.create()
        self.folder_output = self.job.get_folder_output()

        self.job_tmp_folder = self.job.get_folder_tmp()

        if not os.path.exists(self.folder_output):
            os.makedirs(self.folder_output)
        if not os.path.exists(self.job_tmp_folder):
            os.makedirs(self.job_tmp_folder)

        self.logger = get_logger_job(self.job)

        self.container_tag = "joulupukki:" + config['distro'].replace(":", "_")
        self.container = None

    def set_status(self, status):
        self.job.set_status(status)

    def set_build_time(self, build_time):
        self.job.set_build_time(build_time)

    def run(self):
        steps = (('setup', self.setup),
                 ('preparing', self.parse_specdeb),
                 ('save_data', self.save_data),
                 ('building', self.docker_build),
                 ('packaging', self.docker_run),
                 ('finishing', self.get_output),
                 ('cleaning', self.clean_up),
                 )

        for step_name, step_function in steps:
            self.set_status(step_name)
            if step_function() is not True:
                self.logger.debug("Task failed during step: %s", step_name)
                self.set_status('failed')
                return False

        self.set_status('succeeded')
        return True

    def save_data(self):
        if self.config.get('name') is not None and self.builder.build.package_name is None:
            self.builder.build.package_name = self.config.get('name')
        if self.config.get('version') is not None and self.builder.build.package_version is None:
            self.builder.build.package_version = self.config.get('version')
        if self.config.get('release') is not None and self.builder.build.package_release is None:
            self.builder.build.package_release = self.config.get('release')
        self.builder.build._save()
        return True

    def parse_specdeb(self):
        return False

    def docker_build(self):
        return False

    def docker_run(self):
        return False

    def get_output(self):
        return False

    def setup(self):
        # Remove images
        for image in self.cli.images():
            if not any([True for tag in image['RepoTags'] if tag.startswith("joulupukki:")]):
                continue
            try:
                creation_date = datetime.fromtimestamp(image.get('Created'))
                delta_time = datetime.now() - creation_date
                if delta_time.days >= 1:
                    self.logger.debug('Deleting docker image: %s', image['Id'])
                    self.cli.remove_image(image['Id'])
            except Exception as error:
                self.logger.debug('Cannot deleting docker image: %s'
                                  ' - Error: %s', image['Id'], error)
        return True

    def clean_up(self):
        # Set end time
        self.job.set_end_time(time.time())
        # Delete container
        self.logger.debug('Deleting docker container: %s', self.container['Id'])
        self.cli.remove_container(self.container['Id'])
        if os.path.isdir(self.job_tmp_folder):
            shutil.rmtree(self.job_tmp_folder)


        return True
Exemplo n.º 5
0
class Packer(object):
    def __init__(self, builder, config):

        self.config = config

        self.builder = builder
        self.distro = supported_distros[config['distro']]
        self.source_url = builder.source_url
        self.source_type = builder.source_type
        self.cli = builder.cli
        self.folder = builder.folder

        job_data = {
            'distro': config['distro'],
            'username': self.builder.build.username,
            'project_name': self.builder.build.project_name,
            'build_id': self.builder.build.id_,
        }
        self.job = Job(job_data)
        self.job.create()
        self.folder_output = self.job.get_folder_output()

        self.job_tmp_folder = self.job.get_folder_tmp()

        if not os.path.exists(self.folder_output):
            os.makedirs(self.folder_output)
        if not os.path.exists(self.job_tmp_folder):
            os.makedirs(self.job_tmp_folder)

        self.logger = get_logger_job(self.job)

        self.container_tag = "joulupukki:" + config['distro'].replace(":", "_")
        self.container = None

    def set_status(self, status):
        self.job.set_status(status)

    def set_build_time(self, build_time):
        self.job.set_build_time(build_time)

    def run(self):
        steps = (
            ('setup', self.setup),
            ('preparing', self.parse_specdeb),
            ('save_data', self.save_data),
            ('building', self.docker_build),
            ('packaging', self.docker_run),
            ('finishing', self.get_output),
            ('cleaning', self.clean_up),
        )

        for step_name, step_function in steps:
            self.set_status(step_name)
            if step_function() is not True:
                self.logger.debug("Task failed during step: %s", step_name)
                self.set_status('failed')
                return False

        self.set_status('succeeded')
        return True

    def save_data(self):
        if self.config.get(
                'name'
        ) is not None and self.builder.build.package_name is None:
            self.builder.build.package_name = self.config.get('name')
        if self.config.get(
                'version'
        ) is not None and self.builder.build.package_version is None:
            self.builder.build.package_version = self.config.get('version')
        if self.config.get(
                'release'
        ) is not None and self.builder.build.package_release is None:
            self.builder.build.package_release = self.config.get('release')
        self.builder.build._save()
        return True

    def parse_specdeb(self):
        return False

    def docker_build(self):
        return False

    def docker_run(self):
        return False

    def get_output(self):
        return False

    def setup(self):
        # Remove images
        for image in self.cli.images():
            if not any([
                    True for tag in image['RepoTags']
                    if tag.startswith("joulupukki:")
            ]):
                continue
            try:
                creation_date = datetime.fromtimestamp(image.get('Created'))
                delta_time = datetime.now() - creation_date
                if delta_time.days >= 1:
                    self.logger.debug('Deleting docker image: %s', image['Id'])
                    self.cli.remove_image(image['Id'])
            except Exception as error:
                self.logger.debug(
                    'Cannot deleting docker image: %s'
                    ' - Error: %s', image['Id'], error)
        return True

    def clean_up(self):
        # Set end time
        self.job.set_end_time(time.time())
        # Delete container
        self.logger.debug('Deleting docker container: %s',
                          self.container['Id'])
        self.cli.remove_container(self.container['Id'])
        if os.path.isdir(self.job_tmp_folder):
            shutil.rmtree(self.job_tmp_folder)

        return True
Exemplo n.º 6
0
class OsxPacker(object):
    def __init__(self, builder, config):
        self.config = config
        self.builder = builder
        self.distro = "osx"

        self.source_url = builder.source_url
        self.source_type = builder.source_type
        self.branch = builder.build.branch
        self.folder = builder.folder

        self.config['version'] = 'nightly'
        self.config['release'] = '1'
        self.config['name'] = 'ring'

        job_data = {
            'distro': self.distro,
            'username': self.builder.build.username,
            'project_name': self.builder.build.project_name,
            'build_id': self.builder.build.id_,
        }
        self.job = Job(job_data)
        self.job.create()
        self.folder_output = self.job.get_folder_output()

        self.job_tmp_folder = self.job.get_folder_tmp()

        if not os.path.exists(self.folder_output):
            os.makedirs(self.folder_output)
        if not os.path.exists(self.job_tmp_folder):
            os.makedirs(self.job_tmp_folder)

        self.logger = get_logger_job(self.job)

    def set_status(self, status):
        self.job.set_status(status)

    def set_build_time(self, build_time):
        self.job.set_build_time(build_time)

    def run(self):
        steps = (
            ('cloning', self.clone),
            ('reading_conf', self.reading_conf),
            ('setup', self.setup),
            ('compiling', self.compile_),
            ('transfering', self.transfert_output),
        )
        for step_name, step_function in steps:
            self.set_status(step_name)
            if step_function() is not True:
                self.logger.debug("Task failed during step: %s", step_name)
                self.set_status('failed')
                return False
            # Save package name in build.cfg
            if (self.config.get('name') is not None and
                    self.builder.build.package_name is None):
                self.builder.build.package_name = self.config.get('name')
                self.builder.build._save()
        self.set_status('succeeded')
        return True

    def clone(self):
        self.logger.info("Cloning main repo")
        self.logger.info(self.job.get_folder_tmp())
        cmds = [
            "cd %s" % self.job.get_folder_tmp(),
            "git clone -b %s %s source/" % (self.branch, self.source_url),
        ]
        command = " && "
        command = command.join(cmds)

        return self.exec_cmd(command)

    def reading_conf(self):
        self.logger.info("Reading conf from main repo")
        conf_file = "%s/source/.packer.yml" % self.job.get_folder_tmp()
        try:
            stream = open(conf_file, "r")
        except IOError:
            self.logger.error(".packer.yml not present")
            return False
        docs = yaml.load_all(stream)
        osx_conf = {}
        for doc in docs:
            for key, value in doc.items():
                osx_conf[key] = value

        try:
            self.dependencies = osx_conf['osx']['brew_deps']
            self.commands = osx_conf['osx']['commands']
        except KeyError:
            self.logger.error("Malformed .packer.yml file")
            return False
        return True

    def setup(self):
        # Installing dependencies
        for depen in self.dependencies:
            cmd_list = ["brew", "install"]
            cmd_list.extend(depen.split(" "))
            self.logger.info("Installing dependency: %s" % depen)
            process = subprocess.Popen(
                cmd_list,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            stdout, stderr = process.communicate()
            self.logger.debug(stdout)
            self.logger.info(stderr)
            if process.returncode:
                self.logger.error("Error in setup: %d" % process.returncode)
                return False
        return True

    def compile_(self):
        self.logger.info("Start compiling")
        # Compiling ring-daemon
        cd_command = ["cd %s" % self.job.get_folder_tmp()]
        self.commands = cd_command + self.commands
        long_command = " && "
        long_command = long_command.join(self.commands)
        long_command = long_command % {
            "prefix_path": pecan.conf.workspace_path
        }

        self.logger.info("Compiling")
        process = subprocess.Popen(
            long_command,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True
        )
        stdout, stderr = process.communicate()
        self.logger.debug(stdout)
        self.logger.info(stderr)
        if process.returncode:
            self.logger.error("Error in setup: %d" % process.returncode)
            return False
        return True

    def transfert_output(self):
        self.logger.info("Start package transfert")

	# move dmg
        try:
            origin = (self.job.get_folder_tmp() +
                      "/libringclient/ring-client-macosx/build/Ring.dmg")
            destination = (self.builder.build.get_folder_path() +
                           "/output/osx/Ring.dmg")
            os.rename(origin, destination)
        except Exception:
            self.logger.error("Can't move .dmg file")
            return False

        # Delete useless files
        try:
            shutil.rmtree(self.job.get_folder_path() + "/tmp")
        except Exception as e:
            self.logger.error("Couldn't remove tmp job files: " + e)

        host = pecan.conf.origin_host
        user = pecan.conf.origin_user
        key = pecan.conf.origin_key
        # TODO: Correct source and dest (package_dir and path), output/*
        # TODO: Add the transfert of jobs/*
        path = self.builder.origin_build_path
        package_dir = self.builder.build.get_folder_path() + "/*"
        # transfert_command = "scp -r -i %s %s %s@%s:%s" % (
        transfert_command = 'rsync -az -e "ssh -i %s" %s %s@%s:%s --exclude jobs/*/tmp' % (
            key,
            package_dir,
            user,
            host,
            path
        )
        self.logger.info(transfert_command)
        command_res = self.exec_cmd(transfert_command)
        self.logger.info(command_res)
        return command_res

    def clean(self):
        try:
            shutil.rmtree(self.builder.build.get_folder_path())
        except Exception:
            self.logger.error("Could not remove temps files: %s" % (
                self.builder.build.get_folder_path()
            ))
            return False
        return True

    def exec_cmd(self, cmds):
        process = subprocess.Popen(
            cmds,
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            shell=True
        )
        stdout, stderr = process.communicate()
        self.logger.debug(stdout)
        self.logger.info(stderr)
        if process.returncode:
            self.logger.error("Error in setup: %d" % process.returncode)
            return False
        return True