def machines_new(arg_vars, project_root):
print_ok_pending("Creating new Ansible machine profile")
cluster_id = arg_vars['cluster_id']
profile_id = arg_vars['profile_id']
tpl = util.profile_template()
path = util.machine_profile_file(project_root, cluster_id, profile_id)
cf = util.cluster_file(project_root, cluster_id)
contents = yaml.load(open(cf, 'r').read())
region = contents['aws_region']
ami = default_amis[region]
with open(path, "w") as handle:
if (arg_vars.get('services')):
services = [x.strip() for x in arg_vars.get('services').split(",")]
else:
services = []
handle.write(
tpl.render(profile_id=arg_vars['profile_id'],
n_instances=arg_vars['n'],
size=arg_vars['size'],
ami=ami,
services=services))
refresh_provisioning_playbook(arg_vars, project_root)
print_done("Finished Ansible machine profile creation")
def machines_teardown(arg_vars, project_root):
cluster_id = arg_vars['cluster_id']
profile_id = arg_vars['profile_id']
f = util.machine_profile_file(project_root, cluster_id, profile_id)
print_ok_pending("Removing machines provisioned with profile id " +
profile_id)
if verify_profile_exists(arg_vars, project_root):
with open(f, "r") as stream:
content = yaml.load(stream)
content['n_machine_instances'] = 0
with open(f, "w") as stream:
stream.write(yaml.dump(content))
tpl = util.machines_remove_template()
with open(util.machines_remove_file(project_root), "w") as handle:
handle.write(tpl.render(profile=profile_id))
print_ok_pending("Scaling down instances. Streaming Ansible output")
invoke_ansible(arg_vars, project_root, "machines_remove.yml")
call(["rm", f])
print(bcolors.OKBLUE + bcolors.BOLD + "> Finished scale down." +
bcolors.ENDC)
def cluster_teardown(arg_vars, project_root):
print_ok_pending("Tearing down all machine profiles")
print("")
cluster_id = arg_vars['cluster_id']
path = util.machine_profiles_path(project_root, cluster_id)
files = [f for f in listdir(path) if isfile(join(path, f))]
for f in files:
with open(path + "/" + f, 'r') as stream:
content = yaml.load(stream)
arg_vars['profile_id'] = content['profile_id']
machines_teardown(arg_vars, project_root)
print_ok_pending("Tearign down the VPC")
tpl = util.cluster_remove_template()
with open(util.cluster_remove_file(project_root), "w") as text_file:
path = project_root + "/ansible/roles"
services = next(walk(path))[1]
text_file.write(tpl.render(services=services))
invoke_ansible(arg_vars, project_root, util.cluster_remove_playbook())
call(["rm", util.cluster_file(project_root, cluster_id)])
call(["rm", "-r", util.cluster_path(project_root, cluster_id)])
print_done("Finished running Ansible.")
def machines_new(arg_vars, project_root):
print_ok_pending("Creating new Ansible machine profile")
cluster_id = arg_vars['cluster_id']
profile_id = arg_vars['profile_id']
tpl = util.profile_template()
path = util.machine_profile_file(project_root, cluster_id, profile_id)
cf = util.cluster_file(project_root, cluster_id)
contents = yaml.load(open(cf, 'r').read())
region = contents['aws_region']
ami = default_amis[region]
with open(path, "w") as handle:
if (arg_vars.get('services')):
services = [x.strip() for x in arg_vars.get('services').split(",")]
else:
services = []
handle.write(tpl.render(profile_id = arg_vars['profile_id'],
n_instances = arg_vars['n'],
size = arg_vars['size'],
ami = ami,
services = services))
refresh_provisioning_playbook(arg_vars, project_root)
print_done("Finished Ansible machine profile creation")
def cluster_new(arg_vars, project_root):
print_ok_pending("Creating default Ansible playbook")
cluster_id = arg_vars['cluster_id']
call(["mkdir", "-p", util.machine_profiles_path(project_root, cluster_id)])
tpl = util.default_profile_template()
f = util.machine_profile_file(project_root, cluster_id, "default")
region = arg_vars.get('aws_region', 'us-east-1')
ami = default_amis[region]
az = arg_vars.get('aws_az') or default_azs[region]
with open(f, "w") as handle:
handle.write(tpl.render(ami = ami))
tpl = util.common_cluster_template()
f = util.cluster_file(project_root, cluster_id)
with open(f, "w") as text_file:
text_file.write(tpl.render(cluster_id = cluster_id,
aws_region = region,
aws_az = az))
refresh_provisioning_playbook(arg_vars, project_root)
tpl = util.user_post_playbook_template()
f = util.user_post_playbook_file(project_root, cluster_id)
with open(f, "w") as handle:
handle.write(tpl.render())
print_done("Finished Ansible playbook creation.")
print("")
def deploy(arg_vars, project_root):
organization = arg_vars['dockerhub_username']
version = check_output(["lein", "project-version"]).strip()
project_name = os.path.relpath(project_root, "..")
image = organization + "/" + project_name + ":" + version
if not arg_vars.get('skip_uberjar'):
print_ok_pending("Creating an uberjar for your project. Streaming Leiningen output")
call(["lein", "uberjar"])
print_done("Finished creating the uberjar.")
print("")
print_ok_pending("Creating a container. Streaming Docker output")
call(["docker", "build", "-t", image, project_root])
print_done("Finished building container image " + image + ".")
print("")
print_ok_pending("Uploading image to DockerHub")
call(["docker", "push", image])
print_done("Finished pushing image.")
print("")
print_ok_pending("Updating Ansible deployment playbook")
refresh_deployment_playbook(arg_vars, project_root)
print_done("Ansible playbook update complete.")
print("")
print_ok_pending("Running Ansible deployment playbook. Streaming Ansible output")
invoke_ansible(arg_vars, project_root, "deploy.yml",
{"onyx_docker_image": image,
"onyx_tenancy_id": arg_vars['tenancy_id'],
"onyx_n_peers": arg_vars['n_peers']})
print_done("Finished running Ansible. Onyx has been successfully deployed.")
def job_submit(arg_vars, project_root):
print_ok_pending("Invoking Ansible and streaming its output")
refresh_submit_playbook(arg_vars, project_root)
invoke_ansible(arg_vars, project_root,
util.job_submit_playbook(),
{"onyx_job_name": arg_vars['job_name']})
print_done("Finished running Ansible.")
def cluster_provision(arg_vars, project_root):
print_ok_pending("Invoking Ansible and streaming its output")
cluster_id = arg_vars['cluster_id']
refresh_provisioning_playbook(arg_vars, project_root)
invoke_ansible(arg_vars, project_root, util.provisioning_playbook(cluster_id))
invoke_ansible(arg_vars, project_root, util.post_provisioning_playbook(cluster_id))
print_done("Finished running Ansible.")
def machines_cache(arg_vars, project_root):
print_ok_pending("Updating local cache of cluster machines. Streaming Ansible output")
config = ConfigParser.ConfigParser()
engraver_profile = expanduser("~") + "/.engraver"
config.read(engraver_profile)
aws_key_name = config.get('aws', 'aws_key_name', 0)
pem_file_path = config.get('aws', 'pem_file_name', 0)
if(verify_cluster_exists(arg_vars, project_root)):
invoke_ansible(arg_vars, project_root, util.refresh_cache_playbook())
print_done("Finished updating local cache. Displaying cluster: ")
machines_list(arg_vars, project_root, hint=False)
def machines_cache(arg_vars, project_root):
print_ok_pending(
"Updating local cache of cluster machines. Streaming Ansible output")
config = ConfigParser.ConfigParser()
engraver_profile = expanduser("~") + "/.engraver"
config.read(engraver_profile)
aws_key_name = config.get('aws', 'aws_key_name', 0)
pem_file_path = config.get('aws', 'pem_file_name', 0)
if (verify_cluster_exists(arg_vars, project_root)):
invoke_ansible(arg_vars, project_root, util.refresh_cache_playbook())
print_done("Finished updating local cache. Displaying cluster: ")
machines_list(arg_vars, project_root, hint=False)
def deploy(arg_vars, project_root):
organization = arg_vars['dockerhub_username']
version = check_output(["lein", "project-version"]).strip()
project_name = os.path.relpath(project_root, "..")
image = organization + "/" + project_name + ":" + version
if not arg_vars.get('skip_uberjar'):
print_ok_pending(
"Creating an uberjar for your project. Streaming Leiningen output")
call(["lein", "uberjar"])
print_done("Finished creating the uberjar.")
print("")
print_ok_pending("Creating a container. Streaming Docker output")
call(["docker", "build", "-t", image, project_root])
print_done("Finished building container image " + image + ".")
print("")
print_ok_pending("Uploading image to DockerHub")
call(["docker", "push", image])
print_done("Finished pushing image.")
print("")
print_ok_pending("Updating Ansible deployment playbook")
refresh_deployment_playbook(arg_vars, project_root)
print_done("Ansible playbook update complete.")
print("")
print_ok_pending(
"Running Ansible deployment playbook. Streaming Ansible output")
invoke_ansible(
arg_vars, project_root, "deploy.yml", {
"onyx_docker_image": image,
"onyx_tenancy_id": arg_vars['tenancy_id'],
"onyx_n_peers": arg_vars['n_peers']
})
print_done(
"Finished running Ansible. Onyx has been successfully deployed.")
def machines_teardown(arg_vars, project_root):
cluster_id = arg_vars['cluster_id']
profile_id = arg_vars['profile_id']
f = util.machine_profile_file(project_root, cluster_id, profile_id)
print_ok_pending("Removing machines provisioned with profile id " + profile_id)
if verify_profile_exists(arg_vars, project_root):
with open(f, "r") as stream:
content = yaml.load(stream)
content['n_machine_instances'] = 0
with open(f, "w") as stream:
stream.write(yaml.dump(content))
tpl = util.machines_remove_template()
with open(util.machines_remove_file(project_root), "w") as handle:
handle.write(tpl.render(profile = profile_id))
print_ok_pending("Scaling down instances. Streaming Ansible output")
invoke_ansible(arg_vars, project_root, "machines_remove.yml")
call(["rm", f])
print(bcolors.OKBLUE + bcolors.BOLD + "> Finished scale down." + bcolors.ENDC)
def service_pull(arg_vars, project_root):
print_ok_pending("Invoking Git. Streaming its output")
role = util.service_path(project_root, arg_vars['service-name'])
call(["git", "clone", arg_vars['service-repo'], role])
call(["rm", "-rf", role + "/.git"])
print_ok("Installed service to " + role)
def service_new(arg_vars, project_root):
print_ok_pending("Invoking ansible-galaxy. Streaming its output")
role = util.service_path(project_root, arg_vars['service-name'])
call(["ansible-galaxy", "init", role])
print_done("Service role created. Edit at: " + role)
def service_pull(arg_vars, project_root):
print_ok_pending("Invoking Git. Streaming its output")
role = util.service_path(project_root, arg_vars['service-name'])
call(["git", "clone", arg_vars['service-repo'], role])
call(["rm", "-rf", role + "/.git"])
print_ok("Installed service to " + role)
def service_new(arg_vars, project_root):
print_ok_pending("Invoking ansible-galaxy. Streaming its output")
role = util.service_path(project_root, arg_vars['service-name'])
call(["ansible-galaxy", "init", role])
print_done("Service role created. Edit at: " + role)
def init(arg_vars, project_root):
app_name = arg_vars['app-name']
ansible_dir = app_name + "/ansible"
roles_dir = ansible_dir + "/roles"
if arg_vars.get('example_app') == 'beginner':
print_ok_pending("Cloning beginner playbook via Git. Streaming Git output")
call(["git", "clone", "https://github.com/onyx-platform/engraver-beginner-example.git", app_name])
drop_git(app_name)
print_done("Finished cloning example.")
else:
print_ok_pending("Invoking Leiningen and streaming its output")
call(["lein", "new", "onyx-app", app_name, "+docker"])
print_done("Finished executing Leiningen.")
print("")
print_ok_pending("Initializing .engraver folders")
call(["mkdir", "-p", (app_name + "/.engraver")])
call(["touch", (app_name + "/.engraver/config.json")])
print_done("Finished .engraver folder initialization.")
print("")
print_ok_pending("Creating new Ansible playbook. Streaming Ansible output")
call(["ansible-galaxy", "init", ansible_dir])
call(["cp", resource_filename(__name__, "ansible_template/ansible.cfg"), ansible_dir])
call(["cp", resource_filename(__name__, "ansible_template/refresh_cache.yml"), ansible_dir])
print_done("Finished executing Ansible.")
print("")
print_ok_pending("Updating .gitignore for Engraver files")
call("echo '.engraver/clusters/*' >> " + app_name + "/.gitignore", shell=True)
call("echo 'ansible/machines_remove.yml' >> " + app_name + "/.gitignore", shell=True)
call("echo 'ansible/cluster_remove.yml' >> " + app_name + "/.gitignore", shell=True)
call("echo 'ansible/job_submit.yml' >> " + app_name + "/.gitignore", shell=True)
call("echo 'ansible/job_kill.yml' >> " + app_name + "/.gitignore", shell=True)
print_done("Finished updating .gitignore")
print("")
print_ok_pending("Cloning Ansible AWS playbook via Git. Streaming Git output")
path = roles_dir + "/aws"
call(["git", "clone", "https://github.com/onyx-platform/engraver-aws.git", path])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Cloning Ansible Docker playbook via Git. Streaming Git output")
path = roles_dir + "/docker"
call(["git", "clone", "https://github.com/onyx-platform/engraver-docker.git", path])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Cloning Ansible ZooKeeper playbook via Git. Streaming Git output")
path = roles_dir + "/zookeeper"
call(["git", "clone", "https://github.com/onyx-platform/engraver-zookeeper.git", path])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Cloning Ansible BookKeeper playbook via Git. Streaming Git output")
path = roles_dir + "/bookkeeper"
call(["git", "clone", "https://github.com/onyx-platform/engraver-bookkeeper.git", path])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Cloning Ansible Kafka playbook via Git. Streaming Git output")
path = roles_dir + "/kafka"
call(["git", "clone", "https://github.com/onyx-platform/engraver-kafka.git", path])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Cloning Ansible Onyx playbook via Git. Streaming Git output")
path = roles_dir + "/onyx"
call(["git", "clone", "https://github.com/onyx-platform/engraver-onyx.git", path])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Initializing Ansible vars directories")
call(["mkdir", "-p", (ansible_dir + "/group_vars")])
call(["mkdir", "-p", (ansible_dir + "/vars/cluster_vars")])
print_done("Finished Ansible vars directory creation.")
print("")
def init(arg_vars, project_root):
app_name = arg_vars['app-name']
ansible_dir = app_name + "/ansible"
roles_dir = ansible_dir + "/roles"
if arg_vars.get('example_app') == 'beginner':
print_ok_pending(
"Cloning beginner playbook via Git. Streaming Git output")
call([
"git", "clone",
"https://github.com/onyx-platform/engraver-beginner-example.git",
app_name
])
drop_git(app_name)
print_done("Finished cloning example.")
else:
print_ok_pending("Invoking Leiningen and streaming its output")
call(["lein", "new", "onyx-app", app_name, "+docker"])
print_done("Finished executing Leiningen.")
print("")
print_ok_pending("Initializing .engraver folders")
call(["mkdir", "-p", (app_name + "/.engraver")])
call(["touch", (app_name + "/.engraver/config.json")])
print_done("Finished .engraver folder initialization.")
print("")
print_ok_pending("Creating new Ansible playbook. Streaming Ansible output")
call(["ansible-galaxy", "init", ansible_dir])
call([
"cp",
resource_filename(__name__, "ansible_template/ansible.cfg"),
ansible_dir
])
call([
"cp",
resource_filename(__name__, "ansible_template/refresh_cache.yml"),
ansible_dir
])
print_done("Finished executing Ansible.")
print("")
print_ok_pending("Updating .gitignore for Engraver files")
call("echo '.engraver/clusters/*' >> " + app_name + "/.gitignore",
shell=True)
call("echo 'ansible/machines_remove.yml' >> " + app_name + "/.gitignore",
shell=True)
call("echo 'ansible/cluster_remove.yml' >> " + app_name + "/.gitignore",
shell=True)
call("echo 'ansible/job_submit.yml' >> " + app_name + "/.gitignore",
shell=True)
call("echo 'ansible/job_kill.yml' >> " + app_name + "/.gitignore",
shell=True)
print_done("Finished updating .gitignore")
print("")
print_ok_pending(
"Cloning Ansible AWS playbook via Git. Streaming Git output")
path = roles_dir + "/aws"
call([
"git", "clone", "https://github.com/onyx-platform/engraver-aws.git",
path
])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending(
"Cloning Ansible Docker playbook via Git. Streaming Git output")
path = roles_dir + "/docker"
call([
"git", "clone", "https://github.com/onyx-platform/engraver-docker.git",
path
])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending(
"Cloning Ansible ZooKeeper playbook via Git. Streaming Git output")
path = roles_dir + "/zookeeper"
call([
"git", "clone",
"https://github.com/onyx-platform/engraver-zookeeper.git", path
])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending(
"Cloning Ansible BookKeeper playbook via Git. Streaming Git output")
path = roles_dir + "/bookkeeper"
call([
"git", "clone",
"https://github.com/onyx-platform/engraver-bookkeeper.git", path
])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending(
"Cloning Ansible Kafka playbook via Git. Streaming Git output")
path = roles_dir + "/kafka"
call([
"git", "clone", "https://github.com/onyx-platform/engraver-kafka.git",
path
])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending(
"Cloning Ansible Onyx playbook via Git. Streaming Git output")
path = roles_dir + "/onyx"
call([
"git", "clone", "https://github.com/onyx-platform/engraver-onyx.git",
path
])
drop_git(path)
print_done("Finished cloning playbook.")
print("")
print_ok_pending("Initializing Ansible vars directories")
call(["mkdir", "-p", (ansible_dir + "/group_vars")])
call(["mkdir", "-p", (ansible_dir + "/vars/cluster_vars")])
print_done("Finished Ansible vars directory creation.")
print("")
