Exemple #1
0
def init_system(etcdir, vardir):
    """ Initializes the system with system wide config files """

    if etcdir:
        log.info("Initializing config dir %s", etcdir)
        makedirsp(etcdir)
        with open(os.path.join(etcdir, 'config'), 'w') as cfile:
            cfile.write(config.format(os.path.join(etcdir, 'keys'), vardir))

    if vardir:
        log.info("Initializing state dir %s", vardir)
        makedirsp(vardir)
        create_json(os.path.join(vardir, 'info.json'))

    return True
Exemple #2
0
def init_system(etcdir, vardir):
    """ Initializes the system with system wide config files """

    if etcdir:
        log.info("Initializing config dir %s", etcdir)
        makedirsp(etcdir)
        with open(os.path.join(etcdir, 'config'), 'w') as cfile:
            cfile.write(config.format(os.path.join(etcdir, 'keys'), vardir))

    if vardir:
        log.info("Initializing state dir %s", vardir)
        makedirsp(vardir)
        create_json(os.path.join(vardir, 'info.json'))

    return True
Exemple #3
0
def init_environment(squadron_dir, environment_name, copy_from):
    """ Initializes an environment """
    config_dir = os.path.join(squadron_dir, 'config')

    new_env = os.path.join(config_dir, environment_name)

    if copy_from:
        src = os.path.join(config_dir, copy_from)
        shutil.copytree(src, new_env)
    else:
        makedirsp(new_env)
        service_dir = os.path.join(squadron_dir, 'services')
        # Grab all the directories
        to_make = _get_latest_service_versions(service_dir)

        for service_name, service_version in to_make.items():
            create_json(os.path.join(new_env, service_name),
                    {'version': service_version, 'config':{},
                     'base_dir': 'TODO'})

    log.info("Initialized environment {}{}".format(environment_name,
            " copied from " + copy_from if copy_from else ""))
    return True
Exemple #4
0
def init_service(squadron_dir, service_name, service_ver):
    """ Initializes a service with the given name and version """
    service_dir = os.path.join(squadron_dir, 'services', service_name,
                    service_ver)

    rootdir = os.path.join(service_dir, 'root')
    makedirsp(rootdir)
    # rootdir shouldn't be empty
    with open(os.path.join(rootdir, 'config.sq'), 'w') as rfile:
        pass

    testdir = os.path.join(service_dir, 'tests')
    makedirsp(testdir)
    # testdir shouldn't be empty
    with open(os.path.join(testdir, 'example.sh'), 'w') as tfile:
        tfile.write("""#!/bin/bash
while read line; do
    echo $line
done
""")

    # Create the base files
    create_json(os.path.join(service_dir, 'actions'))
    create_json(os.path.join(service_dir, 'defaults'))

    # copy and react's top level are arrays
    create_json(os.path.join(service_dir, 'copy'), [])
    create_json(os.path.join(service_dir, 'react'), [])

    create_json(os.path.join(service_dir, 'schema'), default_schema)
    create_json(os.path.join(service_dir, 'state'), [])

    log.info("Initialized service {} version {}".format(service_name,
            service_ver))

    return True
Exemple #5
0
def init(squadron_dir, gitrepo, force=False, example=False):
    if os.path.exists(squadron_dir):
        # Check if it's empty-ish
        if len(os.listdir(squadron_dir)) > 0 and not force:
            if gitrepo is not None:
                # Grab the gitrepo name out and use that
                repo_name = gitrepo[gitrepo.rstrip('/').rfind('/'):].lstrip('/')
                if repo_name.endswith('.git'):
                    repo_name = repo_name[:-4]

                squadron_dir = os.path.join(squadron_dir, repo_name)
            else:
                log.error("Directory [" + squadron_dir + "] already exists and isn't empty.")
                log.error("Please provide a new directory or use -f.")
                return False

    try:
        if gitrepo is None:
            log.info("Creating Squadron config in {}".format(squadron_dir))
            makedirsp(squadron_dir)
            makedirsp(os.path.join(squadron_dir, 'config'))
            makedirsp(os.path.join(squadron_dir, 'libraries'))
            makedirsp(os.path.join(squadron_dir, 'nodes'))
            makedirsp(os.path.join(squadron_dir, 'resources'))
            makedirsp(os.path.join(squadron_dir, 'services'))
            repo = Repo.init(squadron_dir) # initialize repo
        else:
            log.info("Cloning Squadron config from {}".format(gitrepo))
            repo = Repo.clone_from(gitrepo, squadron_dir)
    except OSError:
        if not _test_for_git():
            log.error("Looks like git isn't installed! Install git to continue")
            return False
        else:
            raise

    log.info("Squadron has been initialized")
    if example is False:
        log.info("Now, you can initialize your service:")
        if squadron_dir != os.getcwd():
            log.info("\tcd %s", squadron_dir)
        log.info("\tsquadron init --service service_name")
    else:
        init_service(squadron_dir, 'example')
        log.info("We have init an example service for you, please check out services/example")

    return True
Exemple #6
0
def _run_squadron(squadron_dir, squadron_state_dir, node_name, dont_rollback,
        force, dry_run):
    """
    Runs apply to set up the temp directory, and then runs commit if
    dry_run is false.

    Keyword arguments:
        squadron_dir -- where the Squadron description dir is
        squadron_state_dir -- where Squadron should store its state between runs
        node_name -- what this node is called
        dont_rollback -- if true, doesn't automatically rollback to the previous version
        force -- treat all files as created, always deploy
        dry_run -- whether or not to apply changes
    """
    log.debug('entering _run_squadron')
    try:
        run_info = runinfo.get_last_run_info(squadron_state_dir, dry_run)
        last_run_dir = run_info['dir']
        last_run_sum = run_info['checksum']
        last_commit = run_info['commit']
    except KeyError:
        log.debug("Looks like info.json is empty or malformated") #Is this bad?
        last_run_dir = None
        last_run_sum = {}
        last_commit = None

    if not dry_run:
        prefix = 'sq-'
        tempdir = os.path.join(squadron_state_dir, 'tmp')
        makedirsp(tempdir)

        new_dir = make_temp(tempdir, prefix, last_run_dir)
    else:
        new_dir = make_temp(tempfile.gettempdir(), 'squadron')

    resources = load_resources(squadron_dir)

    log.info("Staging directory: %s", new_dir)
    result = commit.apply(squadron_dir, node_name, new_dir, resources,
            last_run_dir, dry_run)
    log.debug("commit.apply returned: %s", result)
    # Is this different from the last time we ran?
    this_run_sum = walk_hash(new_dir)

    log.debug("Last run sum: %s", last_run_sum)
    log.debug("This run sum: %s", this_run_sum)

    paths_changed, new_paths = _get_hash_diff(last_run_sum, this_run_sum, force)
    if this_run_sum != last_run_sum or force:
        if not dry_run:
            _deploy(squadron_dir, new_dir, last_run_dir, result,
                    this_run_sum, last_run_sum, last_commit, dont_rollback,
                    resources, force)
            info = {'dir': new_dir, 'commit':result, 'checksum': this_run_sum}
            log.debug("Writing run info to %s: %s", squadron_state_dir, info)

            runinfo.write_run_info(squadron_state_dir, info)

            log.info("Successfully deployed to %s", new_dir)
            return info
        else:
            log.info("Dry run changes")
            actions, reactions = _get_action_reaction(squadron_dir, result)
            log.info("%s actions, %s reactions", len(actions), len(reactions))
            log.debug("Actions are: %s", actions)
            log.debug("Reactions are: %s", reactions)

        log.info("===============")
        log.info("Paths changed:")
        for path in paths_changed:
            log.info("\t%s", path)
        log.info("\nNew paths:")
        for path in new_paths:
            log.info("\t%s", path)
    else:
        if not dry_run:
            _run_actions(squadron_dir, new_dir, result, resources,
                    paths_changed, new_paths)
            _run_tests(squadron_dir, result)
        log.info("Nothing changed.")
    return None
Exemple #7
0
def commit(dir_info):
    """
    Moves files from the temp directory to the final directory based
    on the input given. Returns list of all files

    Keyword arguments:
        dir_info -- dictionary of service to dir_info hash
    """
    def walk_file_list(base_dir, srcdir, resultdir, done_files=set()):
        """ Gets files that haven't been seen yet """
        result = []
        if not base_dir.endswith(os.sep):
            # For stripping the slash
            base_dir = base_dir + os.sep

        for root, dirnames, filenames in os.walk(srcdir):
            after_base = root[len(base_dir):] #strip absolute
            if after_base not in done_files:
                for filename in filenames:
                    if os.path.join(after_base, filename) not in done_files:
                        result.append(os.path.join(resultdir, filename))
        return result

    result = defaultdict(list)
    for service in dir_info:
        # copy the directory
        serv_dir = dir_info[service]['dir']
        base_dir = dir_info[service]['base_dir']

        log.info("Deploying %s to %s", service, base_dir)

        files = set(os.listdir(serv_dir))
        done_files = set()
        for dirname, atomic in dir_info[service]['atomic'].items():
            srcdir = os.path.join(serv_dir, dirname)
            destdir = os.path.join(base_dir, dirname)
            # Delete existing dir
            if atomic:
                if not os.path.islink(destdir):
                    shutil.rmtree(destdir, ignore_errors=True)
                stripped = destdir.rstrip(os.sep)
                makedirsp(os.path.dirname(stripped))
                force_create_symlink(srcdir, stripped)
            else:
                # Copy
                copy_tree(srcdir, destdir)

            result[service].extend(walk_file_list(serv_dir, srcdir, dirname))

            done_files.add(dirname.rstrip(os.sep))

        # Do the remaining files
        for name in files.difference(done_files):
            src = os.path.join(serv_dir, name)
            dst = os.path.join(base_dir, name)
            if os.path.isdir(src):
                if os.path.basename(os.path.normpath(src)) == '.git':
                    continue

                # TODO: Look into how this handles file modes, it's not copying
                # them properly
                _smart_copytree(src, dst, ignore=ignore_copy)
                result[service].extend(walk_file_list(serv_dir, src, name, done_files))
            else:
                # TODO: Look into how this handles file modes, it's not copying
                # them properly
                shutil.copyfile(src, dst)
                result[service].append(name)
    return result
Exemple #8
0
def apply(squadron_dir, node_name, tempdir, resources, previous_run,
        dry_run=False):
    """
    This method takes input from the given squadron_dir and configures
    a temporary directory according to that information

    Keyword arguments:
        squadron_dir -- configuration directory for input
        node_name -- this node's name
        tempdir -- the base temporary directory to use
        previous_run -- the previous successfully applied dir
        dry_run -- whether or not to actually create the temp directory
            or change any system-wide configuration via state file
    """
    log.debug('entering commit.apply %s',
            [squadron_dir, node_name, tempdir, resources, dry_run])
    node_info = get_node_info(os.path.join(squadron_dir, 'nodes'), node_name)

    if not check_node_info(node_info):
        # Return early if there's an error
        log.debug('leaving commit.apply, check_node_info returned error')
        return (False, None)

    log.debug("node_info['env']: " + str(node_info['env']))
    conf_dir = os.path.join(squadron_dir, 'config', node_info['env'])

    result = {}

    # handle the state of the system via the library
    library_dir = os.path.join(squadron_dir, 'libraries')
    state = StateHandler(library_dir)
    for service in node_info['services']:
        # Get config
        configdata = _get_config(conf_dir, service)
        version = configdata['version']
        base_dir = configdata['base_dir']

        get_service_file = functools.partial(_get_service_file, squadron_dir, service, version)

        # defaults file is optional
        cfg = get_service_file('defaults', {})
        cfg.update(configdata['config'])

        # validate each schema
        schema = get_service_file('schema', {})
        if schema:
            jsonschema.validate(cfg, schema)

        # Setting the state comes first, since the rest of this might
        # depend on the state of the system (like virtualenv)
        stateinfo = get_service_file('state', {}, config=cfg)
        for state_item in stateinfo:
            library = state_item['name']
            items = state_item['parameters']

            # Should print these out nicely if they're just strings
            if isinstance(items[0], str) or isinstance(items[0], unicode):
                print_items = ', '.join(items)
            else:
                print_items = items

            log.info("%s %s through %s",
                    "Would process" if dry_run else "Processing",
                    print_items,
                    library)
            state.apply(library, items, dry_run)

        service_dir = os.path.join(squadron_dir, 'services',
                                service, version, 'root')
        render = DirectoryRender(service_dir)

        tmp_serv_dir = os.path.join(tempdir, service)
        makedirsp(tmp_serv_dir)
        # Apply templates
        atomic = render.render(tmp_serv_dir, cfg, resources, dry_run)

        # Copy files from previous runs if applicable
        copy_config = get_service_file('copy', [], config=cfg)
        _apply_copy(copy_config, previous_run, service, tmp_serv_dir)

        result[service] = {
                'atomic': atomic,
                'base_dir': base_dir,
                'config': cfg,
                'dir': tmp_serv_dir,
                'version':version,
            }
    log.debug('leaving commit.apply: ' + str(result))
    return result
Exemple #9
0
def commit(dir_info):
    """
    Moves files from the temp directory to the final directory based
    on the input given. Returns list of all files

    Keyword arguments:
        dir_info -- dictionary of service to dir_info hash
    """
    def walk_file_list(base_dir, srcdir, resultdir, done_files=set()):
        """ Gets files that haven't been seen yet """
        result = []
        if not base_dir.endswith(os.sep):
            # For stripping the slash
            base_dir = base_dir + os.sep

        for root, dirnames, filenames in os.walk(srcdir):
            after_base = root[len(base_dir):]  #strip absolute
            if after_base not in done_files:
                for filename in filenames:
                    if os.path.join(after_base, filename) not in done_files:
                        result.append(os.path.join(resultdir, filename))
        return result

    result = defaultdict(list)
    for service in dir_info:
        # copy the directory
        serv_dir = dir_info[service]['dir']
        base_dir = dir_info[service]['base_dir']

        log.info("Deploying %s to %s", service, base_dir)

        files = set(os.listdir(serv_dir))
        done_files = set()
        for dirname, atomic in dir_info[service]['atomic'].items():
            srcdir = os.path.join(serv_dir, dirname)
            destdir = os.path.join(base_dir, dirname)
            # Delete existing dir
            if atomic:
                if not os.path.islink(destdir):
                    shutil.rmtree(destdir, ignore_errors=True)
                stripped = destdir.rstrip(os.sep)
                makedirsp(os.path.dirname(stripped))
                force_create_symlink(srcdir, stripped)
            else:
                # Copy
                copy_tree(srcdir, destdir)

            result[service].extend(walk_file_list(serv_dir, srcdir, dirname))

            done_files.add(dirname.rstrip(os.sep))

        # Do the remaining files
        for name in files.difference(done_files):
            src = os.path.join(serv_dir, name)
            dst = os.path.join(base_dir, name)
            if os.path.isdir(src):
                if os.path.basename(os.path.normpath(src)) == '.git':
                    continue

                _smart_copytree(src, dst, ignore=ignore_copy)
                result[service].extend(
                    walk_file_list(serv_dir, src, name, done_files))
            else:
                _smart_copyfile(src, dst)
                result[service].append(name)
    return result
Exemple #10
0
def apply(squadron_dir,
          node_name,
          tempdir,
          resources,
          previous_run,
          dry_run=False):
    """
    This method takes input from the given squadron_dir and configures
    a temporary directory according to that information

    Keyword arguments:
        squadron_dir -- configuration directory for input
        node_name -- this node's name
        tempdir -- the base temporary directory to use
        previous_run -- the previous successfully applied dir
        dry_run -- whether or not to actually create the temp directory
            or change any system-wide configuration via state file
    """
    log.debug('entering commit.apply %s',
              [squadron_dir, node_name, tempdir, resources, dry_run])
    node_info = get_node_info(os.path.join(squadron_dir, 'nodes'), node_name)

    if not check_node_info(node_info):
        # Return early if there's an error
        log.debug('leaving commit.apply, check_node_info returned error')
        return (False, None)

    log.debug("node_info['env']: " + str(node_info['env']))
    conf_dir = os.path.join(squadron_dir, 'config', node_info['env'])

    result = {}

    # handle the state of the system via the library
    library_dir = os.path.join(squadron_dir, 'libraries')
    state = StateHandler(library_dir)
    for service in node_info['services']:
        # Get config
        configdata = _get_config(conf_dir, service)
        version = configdata['version']
        base_dir = configdata['base_dir']

        get_service_file = functools.partial(_get_service_file, squadron_dir,
                                             service, version)

        # defaults file is optional
        cfg = get_service_file('defaults', {})
        cfg.update({'node_name': node_name})
        cfg.update(configdata['config'])

        # validate each schema
        schema = get_service_file('schema', {})
        if schema:
            jsonschema.validate(cfg, schema)

        # Setting the state comes first, since the rest of this might
        # depend on the state of the system (like virtualenv)
        stateinfo = get_service_file('state', {}, config=cfg)
        for state_item in stateinfo:
            library = state_item['name']
            items = state_item['parameters']

            # Should print these out nicely if they're just strings
            if isinstance(items[0], str) or isinstance(items[0], unicode):
                print_items = ', '.join(items)
            else:
                print_items = items

            log.info("%s %s through %s",
                     "Would process" if dry_run else "Processing", print_items,
                     library)
            state.apply(library, items, dry_run)

        service_dir = os.path.join(squadron_dir, 'services', service, version,
                                   'root')
        render = DirectoryRender(service_dir)

        tmp_serv_dir = os.path.join(tempdir, service)
        makedirsp(tmp_serv_dir)
        # Apply templates
        atomic = render.render(tmp_serv_dir, cfg, resources, dry_run)

        # Copy files from previous runs if applicable
        copy_config = get_service_file('copy', [], config=cfg)
        _apply_copy(copy_config, previous_run, service, tmp_serv_dir)

        result[service] = {
            'atomic': atomic,
            'base_dir': base_dir,
            'config': cfg,
            'dir': tmp_serv_dir,
            'version': version,
        }
    log.debug('leaving commit.apply: ' + str(result))
    return result