def init_system(etcdir, vardir):
""" Initializes the system with system wide config files """
if etcdir:
log.info("Initializing config dir %s", etcdir)
makedirsp(etcdir)
with open(os.path.join(etcdir, 'config'), 'w') as cfile:
cfile.write(config.format(os.path.join(etcdir, 'keys'), vardir))
if vardir:
log.info("Initializing state dir %s", vardir)
makedirsp(vardir)
create_json(os.path.join(vardir, 'info.json'))
return True
def init_system(etcdir, vardir):
""" Initializes the system with system wide config files """
if etcdir:
log.info("Initializing config dir %s", etcdir)
makedirsp(etcdir)
with open(os.path.join(etcdir, 'config'), 'w') as cfile:
cfile.write(config.format(os.path.join(etcdir, 'keys'), vardir))
if vardir:
log.info("Initializing state dir %s", vardir)
makedirsp(vardir)
create_json(os.path.join(vardir, 'info.json'))
return True
def init_environment(squadron_dir, environment_name, copy_from):
""" Initializes an environment """
config_dir = os.path.join(squadron_dir, 'config')
new_env = os.path.join(config_dir, environment_name)
if copy_from:
src = os.path.join(config_dir, copy_from)
shutil.copytree(src, new_env)
else:
makedirsp(new_env)
service_dir = os.path.join(squadron_dir, 'services')
# Grab all the directories
to_make = _get_latest_service_versions(service_dir)
for service_name, service_version in to_make.items():
create_json(os.path.join(new_env, service_name),
{'version': service_version, 'config':{},
'base_dir': 'TODO'})
log.info("Initialized environment {}{}".format(environment_name,
" copied from " + copy_from if copy_from else ""))
return True
def init_service(squadron_dir, service_name, service_ver):
""" Initializes a service with the given name and version """
service_dir = os.path.join(squadron_dir, 'services', service_name,
service_ver)
rootdir = os.path.join(service_dir, 'root')
makedirsp(rootdir)
# rootdir shouldn't be empty
with open(os.path.join(rootdir, 'config.sq'), 'w') as rfile:
pass
testdir = os.path.join(service_dir, 'tests')
makedirsp(testdir)
# testdir shouldn't be empty
with open(os.path.join(testdir, 'example.sh'), 'w') as tfile:
tfile.write("""#!/bin/bash
while read line; do
echo $line
done
""")
# Create the base files
create_json(os.path.join(service_dir, 'actions'))
create_json(os.path.join(service_dir, 'defaults'))
# copy and react's top level are arrays
create_json(os.path.join(service_dir, 'copy'), [])
create_json(os.path.join(service_dir, 'react'), [])
create_json(os.path.join(service_dir, 'schema'), default_schema)
create_json(os.path.join(service_dir, 'state'), [])
log.info("Initialized service {} version {}".format(service_name,
service_ver))
return True
def init(squadron_dir, gitrepo, force=False, example=False):
if os.path.exists(squadron_dir):
# Check if it's empty-ish
if len(os.listdir(squadron_dir)) > 0 and not force:
if gitrepo is not None:
# Grab the gitrepo name out and use that
repo_name = gitrepo[gitrepo.rstrip('/').rfind('/'):].lstrip('/')
if repo_name.endswith('.git'):
repo_name = repo_name[:-4]
squadron_dir = os.path.join(squadron_dir, repo_name)
else:
log.error("Directory [" + squadron_dir + "] already exists and isn't empty.")
log.error("Please provide a new directory or use -f.")
return False
try:
if gitrepo is None:
log.info("Creating Squadron config in {}".format(squadron_dir))
makedirsp(squadron_dir)
makedirsp(os.path.join(squadron_dir, 'config'))
makedirsp(os.path.join(squadron_dir, 'libraries'))
makedirsp(os.path.join(squadron_dir, 'nodes'))
makedirsp(os.path.join(squadron_dir, 'resources'))
makedirsp(os.path.join(squadron_dir, 'services'))
repo = Repo.init(squadron_dir) # initialize repo
else:
log.info("Cloning Squadron config from {}".format(gitrepo))
repo = Repo.clone_from(gitrepo, squadron_dir)
except OSError:
if not _test_for_git():
log.error("Looks like git isn't installed! Install git to continue")
return False
else:
raise
log.info("Squadron has been initialized")
if example is False:
log.info("Now, you can initialize your service:")
if squadron_dir != os.getcwd():
log.info("\tcd %s", squadron_dir)
log.info("\tsquadron init --service service_name")
else:
init_service(squadron_dir, 'example')
log.info("We have init an example service for you, please check out services/example")
return True
def _run_squadron(squadron_dir, squadron_state_dir, node_name, dont_rollback,
force, dry_run):
"""
Runs apply to set up the temp directory, and then runs commit if
dry_run is false.
Keyword arguments:
squadron_dir -- where the Squadron description dir is
squadron_state_dir -- where Squadron should store its state between runs
node_name -- what this node is called
dont_rollback -- if true, doesn't automatically rollback to the previous version
force -- treat all files as created, always deploy
dry_run -- whether or not to apply changes
"""
log.debug('entering _run_squadron')
try:
run_info = runinfo.get_last_run_info(squadron_state_dir, dry_run)
last_run_dir = run_info['dir']
last_run_sum = run_info['checksum']
last_commit = run_info['commit']
except KeyError:
log.debug("Looks like info.json is empty or malformated") #Is this bad?
last_run_dir = None
last_run_sum = {}
last_commit = None
if not dry_run:
prefix = 'sq-'
tempdir = os.path.join(squadron_state_dir, 'tmp')
makedirsp(tempdir)
new_dir = make_temp(tempdir, prefix, last_run_dir)
else:
new_dir = make_temp(tempfile.gettempdir(), 'squadron')
resources = load_resources(squadron_dir)
log.info("Staging directory: %s", new_dir)
result = commit.apply(squadron_dir, node_name, new_dir, resources,
last_run_dir, dry_run)
log.debug("commit.apply returned: %s", result)
# Is this different from the last time we ran?
this_run_sum = walk_hash(new_dir)
log.debug("Last run sum: %s", last_run_sum)
log.debug("This run sum: %s", this_run_sum)
paths_changed, new_paths = _get_hash_diff(last_run_sum, this_run_sum, force)
if this_run_sum != last_run_sum or force:
if not dry_run:
_deploy(squadron_dir, new_dir, last_run_dir, result,
this_run_sum, last_run_sum, last_commit, dont_rollback,
resources, force)
info = {'dir': new_dir, 'commit':result, 'checksum': this_run_sum}
log.debug("Writing run info to %s: %s", squadron_state_dir, info)
runinfo.write_run_info(squadron_state_dir, info)
log.info("Successfully deployed to %s", new_dir)
return info
else:
log.info("Dry run changes")
actions, reactions = _get_action_reaction(squadron_dir, result)
log.info("%s actions, %s reactions", len(actions), len(reactions))
log.debug("Actions are: %s", actions)
log.debug("Reactions are: %s", reactions)
log.info("===============")
log.info("Paths changed:")
for path in paths_changed:
log.info("\t%s", path)
log.info("\nNew paths:")
for path in new_paths:
log.info("\t%s", path)
else:
if not dry_run:
_run_actions(squadron_dir, new_dir, result, resources,
paths_changed, new_paths)
_run_tests(squadron_dir, result)
log.info("Nothing changed.")
return None
def commit(dir_info):
"""
Moves files from the temp directory to the final directory based
on the input given. Returns list of all files
Keyword arguments:
dir_info -- dictionary of service to dir_info hash
"""
def walk_file_list(base_dir, srcdir, resultdir, done_files=set()):
""" Gets files that haven't been seen yet """
result = []
if not base_dir.endswith(os.sep):
# For stripping the slash
base_dir = base_dir + os.sep
for root, dirnames, filenames in os.walk(srcdir):
after_base = root[len(base_dir):] #strip absolute
if after_base not in done_files:
for filename in filenames:
if os.path.join(after_base, filename) not in done_files:
result.append(os.path.join(resultdir, filename))
return result
result = defaultdict(list)
for service in dir_info:
# copy the directory
serv_dir = dir_info[service]['dir']
base_dir = dir_info[service]['base_dir']
log.info("Deploying %s to %s", service, base_dir)
files = set(os.listdir(serv_dir))
done_files = set()
for dirname, atomic in dir_info[service]['atomic'].items():
srcdir = os.path.join(serv_dir, dirname)
destdir = os.path.join(base_dir, dirname)
# Delete existing dir
if atomic:
if not os.path.islink(destdir):
shutil.rmtree(destdir, ignore_errors=True)
stripped = destdir.rstrip(os.sep)
makedirsp(os.path.dirname(stripped))
force_create_symlink(srcdir, stripped)
else:
# Copy
copy_tree(srcdir, destdir)
result[service].extend(walk_file_list(serv_dir, srcdir, dirname))
done_files.add(dirname.rstrip(os.sep))
# Do the remaining files
for name in files.difference(done_files):
src = os.path.join(serv_dir, name)
dst = os.path.join(base_dir, name)
if os.path.isdir(src):
if os.path.basename(os.path.normpath(src)) == '.git':
continue
# TODO: Look into how this handles file modes, it's not copying
# them properly
_smart_copytree(src, dst, ignore=ignore_copy)
result[service].extend(walk_file_list(serv_dir, src, name, done_files))
else:
# TODO: Look into how this handles file modes, it's not copying
# them properly
shutil.copyfile(src, dst)
result[service].append(name)
return result
def apply(squadron_dir, node_name, tempdir, resources, previous_run,
dry_run=False):
"""
This method takes input from the given squadron_dir and configures
a temporary directory according to that information
Keyword arguments:
squadron_dir -- configuration directory for input
node_name -- this node's name
tempdir -- the base temporary directory to use
previous_run -- the previous successfully applied dir
dry_run -- whether or not to actually create the temp directory
or change any system-wide configuration via state file
"""
log.debug('entering commit.apply %s',
[squadron_dir, node_name, tempdir, resources, dry_run])
node_info = get_node_info(os.path.join(squadron_dir, 'nodes'), node_name)
if not check_node_info(node_info):
# Return early if there's an error
log.debug('leaving commit.apply, check_node_info returned error')
return (False, None)
log.debug("node_info['env']: " + str(node_info['env']))
conf_dir = os.path.join(squadron_dir, 'config', node_info['env'])
result = {}
# handle the state of the system via the library
library_dir = os.path.join(squadron_dir, 'libraries')
state = StateHandler(library_dir)
for service in node_info['services']:
# Get config
configdata = _get_config(conf_dir, service)
version = configdata['version']
base_dir = configdata['base_dir']
get_service_file = functools.partial(_get_service_file, squadron_dir, service, version)
# defaults file is optional
cfg = get_service_file('defaults', {})
cfg.update(configdata['config'])
# validate each schema
schema = get_service_file('schema', {})
if schema:
jsonschema.validate(cfg, schema)
# Setting the state comes first, since the rest of this might
# depend on the state of the system (like virtualenv)
stateinfo = get_service_file('state', {}, config=cfg)
for state_item in stateinfo:
library = state_item['name']
items = state_item['parameters']
# Should print these out nicely if they're just strings
if isinstance(items[0], str) or isinstance(items[0], unicode):
print_items = ', '.join(items)
else:
print_items = items
log.info("%s %s through %s",
"Would process" if dry_run else "Processing",
print_items,
library)
state.apply(library, items, dry_run)
service_dir = os.path.join(squadron_dir, 'services',
service, version, 'root')
render = DirectoryRender(service_dir)
tmp_serv_dir = os.path.join(tempdir, service)
makedirsp(tmp_serv_dir)
# Apply templates
atomic = render.render(tmp_serv_dir, cfg, resources, dry_run)
# Copy files from previous runs if applicable
copy_config = get_service_file('copy', [], config=cfg)
_apply_copy(copy_config, previous_run, service, tmp_serv_dir)
result[service] = {
'atomic': atomic,
'base_dir': base_dir,
'config': cfg,
'dir': tmp_serv_dir,
'version':version,
}
log.debug('leaving commit.apply: ' + str(result))
return result
def commit(dir_info):
"""
Moves files from the temp directory to the final directory based
on the input given. Returns list of all files
Keyword arguments:
dir_info -- dictionary of service to dir_info hash
"""
def walk_file_list(base_dir, srcdir, resultdir, done_files=set()):
""" Gets files that haven't been seen yet """
result = []
if not base_dir.endswith(os.sep):
# For stripping the slash
base_dir = base_dir + os.sep
for root, dirnames, filenames in os.walk(srcdir):
after_base = root[len(base_dir):] #strip absolute
if after_base not in done_files:
for filename in filenames:
if os.path.join(after_base, filename) not in done_files:
result.append(os.path.join(resultdir, filename))
return result
result = defaultdict(list)
for service in dir_info:
# copy the directory
serv_dir = dir_info[service]['dir']
base_dir = dir_info[service]['base_dir']
log.info("Deploying %s to %s", service, base_dir)
files = set(os.listdir(serv_dir))
done_files = set()
for dirname, atomic in dir_info[service]['atomic'].items():
srcdir = os.path.join(serv_dir, dirname)
destdir = os.path.join(base_dir, dirname)
# Delete existing dir
if atomic:
if not os.path.islink(destdir):
shutil.rmtree(destdir, ignore_errors=True)
stripped = destdir.rstrip(os.sep)
makedirsp(os.path.dirname(stripped))
force_create_symlink(srcdir, stripped)
else:
# Copy
copy_tree(srcdir, destdir)
result[service].extend(walk_file_list(serv_dir, srcdir, dirname))
done_files.add(dirname.rstrip(os.sep))
# Do the remaining files
for name in files.difference(done_files):
src = os.path.join(serv_dir, name)
dst = os.path.join(base_dir, name)
if os.path.isdir(src):
if os.path.basename(os.path.normpath(src)) == '.git':
continue
_smart_copytree(src, dst, ignore=ignore_copy)
result[service].extend(
walk_file_list(serv_dir, src, name, done_files))
else:
_smart_copyfile(src, dst)
result[service].append(name)
return result
def apply(squadron_dir,
node_name,
tempdir,
resources,
previous_run,
dry_run=False):
"""
This method takes input from the given squadron_dir and configures
a temporary directory according to that information
Keyword arguments:
squadron_dir -- configuration directory for input
node_name -- this node's name
tempdir -- the base temporary directory to use
previous_run -- the previous successfully applied dir
dry_run -- whether or not to actually create the temp directory
or change any system-wide configuration via state file
"""
log.debug('entering commit.apply %s',
[squadron_dir, node_name, tempdir, resources, dry_run])
node_info = get_node_info(os.path.join(squadron_dir, 'nodes'), node_name)
if not check_node_info(node_info):
# Return early if there's an error
log.debug('leaving commit.apply, check_node_info returned error')
return (False, None)
log.debug("node_info['env']: " + str(node_info['env']))
conf_dir = os.path.join(squadron_dir, 'config', node_info['env'])
result = {}
# handle the state of the system via the library
library_dir = os.path.join(squadron_dir, 'libraries')
state = StateHandler(library_dir)
for service in node_info['services']:
# Get config
configdata = _get_config(conf_dir, service)
version = configdata['version']
base_dir = configdata['base_dir']
get_service_file = functools.partial(_get_service_file, squadron_dir,
service, version)
# defaults file is optional
cfg = get_service_file('defaults', {})
cfg.update({'node_name': node_name})
cfg.update(configdata['config'])
# validate each schema
schema = get_service_file('schema', {})
if schema:
jsonschema.validate(cfg, schema)
# Setting the state comes first, since the rest of this might
# depend on the state of the system (like virtualenv)
stateinfo = get_service_file('state', {}, config=cfg)
for state_item in stateinfo:
library = state_item['name']
items = state_item['parameters']
# Should print these out nicely if they're just strings
if isinstance(items[0], str) or isinstance(items[0], unicode):
print_items = ', '.join(items)
else:
print_items = items
log.info("%s %s through %s",
"Would process" if dry_run else "Processing", print_items,
library)
state.apply(library, items, dry_run)
service_dir = os.path.join(squadron_dir, 'services', service, version,
'root')
render = DirectoryRender(service_dir)
tmp_serv_dir = os.path.join(tempdir, service)
makedirsp(tmp_serv_dir)
# Apply templates
atomic = render.render(tmp_serv_dir, cfg, resources, dry_run)
# Copy files from previous runs if applicable
copy_config = get_service_file('copy', [], config=cfg)
_apply_copy(copy_config, previous_run, service, tmp_serv_dir)
result[service] = {
'atomic': atomic,
'base_dir': base_dir,
'config': cfg,
'dir': tmp_serv_dir,
'version': version,
}
log.debug('leaving commit.apply: ' + str(result))
return result
