def machines_new(arg_vars, project_root): print_ok_pending("Creating new Ansible machine profile") cluster_id = arg_vars['cluster_id'] profile_id = arg_vars['profile_id'] tpl = util.profile_template() path = util.machine_profile_file(project_root, cluster_id, profile_id) cf = util.cluster_file(project_root, cluster_id) contents = yaml.load(open(cf, 'r').read()) region = contents['aws_region'] ami = default_amis[region] with open(path, "w") as handle: if (arg_vars.get('services')): services = [x.strip() for x in arg_vars.get('services').split(",")] else: services = [] handle.write( tpl.render(profile_id=arg_vars['profile_id'], n_instances=arg_vars['n'], size=arg_vars['size'], ami=ami, services=services)) refresh_provisioning_playbook(arg_vars, project_root) print_done("Finished Ansible machine profile creation")
def machines_teardown(arg_vars, project_root): cluster_id = arg_vars['cluster_id'] profile_id = arg_vars['profile_id'] f = util.machine_profile_file(project_root, cluster_id, profile_id) print_ok_pending("Removing machines provisioned with profile id " + profile_id) if verify_profile_exists(arg_vars, project_root): with open(f, "r") as stream: content = yaml.load(stream) content['n_machine_instances'] = 0 with open(f, "w") as stream: stream.write(yaml.dump(content)) tpl = util.machines_remove_template() with open(util.machines_remove_file(project_root), "w") as handle: handle.write(tpl.render(profile=profile_id)) print_ok_pending("Scaling down instances. Streaming Ansible output") invoke_ansible(arg_vars, project_root, "machines_remove.yml") call(["rm", f]) print(bcolors.OKBLUE + bcolors.BOLD + "> Finished scale down." + bcolors.ENDC)
def cluster_teardown(arg_vars, project_root): print_ok_pending("Tearing down all machine profiles") print("") cluster_id = arg_vars['cluster_id'] path = util.machine_profiles_path(project_root, cluster_id) files = [f for f in listdir(path) if isfile(join(path, f))] for f in files: with open(path + "/" + f, 'r') as stream: content = yaml.load(stream) arg_vars['profile_id'] = content['profile_id'] machines_teardown(arg_vars, project_root) print_ok_pending("Tearign down the VPC") tpl = util.cluster_remove_template() with open(util.cluster_remove_file(project_root), "w") as text_file: path = project_root + "/ansible/roles" services = next(walk(path))[1] text_file.write(tpl.render(services=services)) invoke_ansible(arg_vars, project_root, util.cluster_remove_playbook()) call(["rm", util.cluster_file(project_root, cluster_id)]) call(["rm", "-r", util.cluster_path(project_root, cluster_id)]) print_done("Finished running Ansible.")
def machines_new(arg_vars, project_root): print_ok_pending("Creating new Ansible machine profile") cluster_id = arg_vars['cluster_id'] profile_id = arg_vars['profile_id'] tpl = util.profile_template() path = util.machine_profile_file(project_root, cluster_id, profile_id) cf = util.cluster_file(project_root, cluster_id) contents = yaml.load(open(cf, 'r').read()) region = contents['aws_region'] ami = default_amis[region] with open(path, "w") as handle: if (arg_vars.get('services')): services = [x.strip() for x in arg_vars.get('services').split(",")] else: services = [] handle.write(tpl.render(profile_id = arg_vars['profile_id'], n_instances = arg_vars['n'], size = arg_vars['size'], ami = ami, services = services)) refresh_provisioning_playbook(arg_vars, project_root) print_done("Finished Ansible machine profile creation")
def cluster_new(arg_vars, project_root): print_ok_pending("Creating default Ansible playbook") cluster_id = arg_vars['cluster_id'] call(["mkdir", "-p", util.machine_profiles_path(project_root, cluster_id)]) tpl = util.default_profile_template() f = util.machine_profile_file(project_root, cluster_id, "default") region = arg_vars.get('aws_region', 'us-east-1') ami = default_amis[region] az = arg_vars.get('aws_az') or default_azs[region] with open(f, "w") as handle: handle.write(tpl.render(ami = ami)) tpl = util.common_cluster_template() f = util.cluster_file(project_root, cluster_id) with open(f, "w") as text_file: text_file.write(tpl.render(cluster_id = cluster_id, aws_region = region, aws_az = az)) refresh_provisioning_playbook(arg_vars, project_root) tpl = util.user_post_playbook_template() f = util.user_post_playbook_file(project_root, cluster_id) with open(f, "w") as handle: handle.write(tpl.render()) print_done("Finished Ansible playbook creation.") print("")
def deploy(arg_vars, project_root): organization = arg_vars['dockerhub_username'] version = check_output(["lein", "project-version"]).strip() project_name = os.path.relpath(project_root, "..") image = organization + "/" + project_name + ":" + version if not arg_vars.get('skip_uberjar'): print_ok_pending("Creating an uberjar for your project. Streaming Leiningen output") call(["lein", "uberjar"]) print_done("Finished creating the uberjar.") print("") print_ok_pending("Creating a container. Streaming Docker output") call(["docker", "build", "-t", image, project_root]) print_done("Finished building container image " + image + ".") print("") print_ok_pending("Uploading image to DockerHub") call(["docker", "push", image]) print_done("Finished pushing image.") print("") print_ok_pending("Updating Ansible deployment playbook") refresh_deployment_playbook(arg_vars, project_root) print_done("Ansible playbook update complete.") print("") print_ok_pending("Running Ansible deployment playbook. Streaming Ansible output") invoke_ansible(arg_vars, project_root, "deploy.yml", {"onyx_docker_image": image, "onyx_tenancy_id": arg_vars['tenancy_id'], "onyx_n_peers": arg_vars['n_peers']}) print_done("Finished running Ansible. Onyx has been successfully deployed.")
def job_submit(arg_vars, project_root): print_ok_pending("Invoking Ansible and streaming its output") refresh_submit_playbook(arg_vars, project_root) invoke_ansible(arg_vars, project_root, util.job_submit_playbook(), {"onyx_job_name": arg_vars['job_name']}) print_done("Finished running Ansible.")
def cluster_provision(arg_vars, project_root): print_ok_pending("Invoking Ansible and streaming its output") cluster_id = arg_vars['cluster_id'] refresh_provisioning_playbook(arg_vars, project_root) invoke_ansible(arg_vars, project_root, util.provisioning_playbook(cluster_id)) invoke_ansible(arg_vars, project_root, util.post_provisioning_playbook(cluster_id)) print_done("Finished running Ansible.")
def machines_cache(arg_vars, project_root): print_ok_pending("Updating local cache of cluster machines. Streaming Ansible output") config = ConfigParser.ConfigParser() engraver_profile = expanduser("~") + "/.engraver" config.read(engraver_profile) aws_key_name = config.get('aws', 'aws_key_name', 0) pem_file_path = config.get('aws', 'pem_file_name', 0) if(verify_cluster_exists(arg_vars, project_root)): invoke_ansible(arg_vars, project_root, util.refresh_cache_playbook()) print_done("Finished updating local cache. Displaying cluster: ") machines_list(arg_vars, project_root, hint=False)
def machines_cache(arg_vars, project_root): print_ok_pending( "Updating local cache of cluster machines. Streaming Ansible output") config = ConfigParser.ConfigParser() engraver_profile = expanduser("~") + "/.engraver" config.read(engraver_profile) aws_key_name = config.get('aws', 'aws_key_name', 0) pem_file_path = config.get('aws', 'pem_file_name', 0) if (verify_cluster_exists(arg_vars, project_root)): invoke_ansible(arg_vars, project_root, util.refresh_cache_playbook()) print_done("Finished updating local cache. Displaying cluster: ") machines_list(arg_vars, project_root, hint=False)
def deploy(arg_vars, project_root): organization = arg_vars['dockerhub_username'] version = check_output(["lein", "project-version"]).strip() project_name = os.path.relpath(project_root, "..") image = organization + "/" + project_name + ":" + version if not arg_vars.get('skip_uberjar'): print_ok_pending( "Creating an uberjar for your project. Streaming Leiningen output") call(["lein", "uberjar"]) print_done("Finished creating the uberjar.") print("") print_ok_pending("Creating a container. Streaming Docker output") call(["docker", "build", "-t", image, project_root]) print_done("Finished building container image " + image + ".") print("") print_ok_pending("Uploading image to DockerHub") call(["docker", "push", image]) print_done("Finished pushing image.") print("") print_ok_pending("Updating Ansible deployment playbook") refresh_deployment_playbook(arg_vars, project_root) print_done("Ansible playbook update complete.") print("") print_ok_pending( "Running Ansible deployment playbook. Streaming Ansible output") invoke_ansible( arg_vars, project_root, "deploy.yml", { "onyx_docker_image": image, "onyx_tenancy_id": arg_vars['tenancy_id'], "onyx_n_peers": arg_vars['n_peers'] }) print_done( "Finished running Ansible. Onyx has been successfully deployed.")
def machines_teardown(arg_vars, project_root): cluster_id = arg_vars['cluster_id'] profile_id = arg_vars['profile_id'] f = util.machine_profile_file(project_root, cluster_id, profile_id) print_ok_pending("Removing machines provisioned with profile id " + profile_id) if verify_profile_exists(arg_vars, project_root): with open(f, "r") as stream: content = yaml.load(stream) content['n_machine_instances'] = 0 with open(f, "w") as stream: stream.write(yaml.dump(content)) tpl = util.machines_remove_template() with open(util.machines_remove_file(project_root), "w") as handle: handle.write(tpl.render(profile = profile_id)) print_ok_pending("Scaling down instances. Streaming Ansible output") invoke_ansible(arg_vars, project_root, "machines_remove.yml") call(["rm", f]) print(bcolors.OKBLUE + bcolors.BOLD + "> Finished scale down." + bcolors.ENDC)
def service_pull(arg_vars, project_root): print_ok_pending("Invoking Git. Streaming its output") role = util.service_path(project_root, arg_vars['service-name']) call(["git", "clone", arg_vars['service-repo'], role]) call(["rm", "-rf", role + "/.git"]) print_ok("Installed service to " + role)
def service_new(arg_vars, project_root): print_ok_pending("Invoking ansible-galaxy. Streaming its output") role = util.service_path(project_root, arg_vars['service-name']) call(["ansible-galaxy", "init", role]) print_done("Service role created. Edit at: " + role)
def init(arg_vars, project_root): app_name = arg_vars['app-name'] ansible_dir = app_name + "/ansible" roles_dir = ansible_dir + "/roles" if arg_vars.get('example_app') == 'beginner': print_ok_pending("Cloning beginner playbook via Git. Streaming Git output") call(["git", "clone", "https://github.com/onyx-platform/engraver-beginner-example.git", app_name]) drop_git(app_name) print_done("Finished cloning example.") else: print_ok_pending("Invoking Leiningen and streaming its output") call(["lein", "new", "onyx-app", app_name, "+docker"]) print_done("Finished executing Leiningen.") print("") print_ok_pending("Initializing .engraver folders") call(["mkdir", "-p", (app_name + "/.engraver")]) call(["touch", (app_name + "/.engraver/config.json")]) print_done("Finished .engraver folder initialization.") print("") print_ok_pending("Creating new Ansible playbook. Streaming Ansible output") call(["ansible-galaxy", "init", ansible_dir]) call(["cp", resource_filename(__name__, "ansible_template/ansible.cfg"), ansible_dir]) call(["cp", resource_filename(__name__, "ansible_template/refresh_cache.yml"), ansible_dir]) print_done("Finished executing Ansible.") print("") print_ok_pending("Updating .gitignore for Engraver files") call("echo '.engraver/clusters/*' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/machines_remove.yml' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/cluster_remove.yml' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/job_submit.yml' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/job_kill.yml' >> " + app_name + "/.gitignore", shell=True) print_done("Finished updating .gitignore") print("") print_ok_pending("Cloning Ansible AWS playbook via Git. Streaming Git output") path = roles_dir + "/aws" call(["git", "clone", "https://github.com/onyx-platform/engraver-aws.git", path]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Cloning Ansible Docker playbook via Git. Streaming Git output") path = roles_dir + "/docker" call(["git", "clone", "https://github.com/onyx-platform/engraver-docker.git", path]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Cloning Ansible ZooKeeper playbook via Git. Streaming Git output") path = roles_dir + "/zookeeper" call(["git", "clone", "https://github.com/onyx-platform/engraver-zookeeper.git", path]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Cloning Ansible BookKeeper playbook via Git. Streaming Git output") path = roles_dir + "/bookkeeper" call(["git", "clone", "https://github.com/onyx-platform/engraver-bookkeeper.git", path]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Cloning Ansible Kafka playbook via Git. Streaming Git output") path = roles_dir + "/kafka" call(["git", "clone", "https://github.com/onyx-platform/engraver-kafka.git", path]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Cloning Ansible Onyx playbook via Git. Streaming Git output") path = roles_dir + "/onyx" call(["git", "clone", "https://github.com/onyx-platform/engraver-onyx.git", path]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Initializing Ansible vars directories") call(["mkdir", "-p", (ansible_dir + "/group_vars")]) call(["mkdir", "-p", (ansible_dir + "/vars/cluster_vars")]) print_done("Finished Ansible vars directory creation.") print("")
def init(arg_vars, project_root): app_name = arg_vars['app-name'] ansible_dir = app_name + "/ansible" roles_dir = ansible_dir + "/roles" if arg_vars.get('example_app') == 'beginner': print_ok_pending( "Cloning beginner playbook via Git. Streaming Git output") call([ "git", "clone", "https://github.com/onyx-platform/engraver-beginner-example.git", app_name ]) drop_git(app_name) print_done("Finished cloning example.") else: print_ok_pending("Invoking Leiningen and streaming its output") call(["lein", "new", "onyx-app", app_name, "+docker"]) print_done("Finished executing Leiningen.") print("") print_ok_pending("Initializing .engraver folders") call(["mkdir", "-p", (app_name + "/.engraver")]) call(["touch", (app_name + "/.engraver/config.json")]) print_done("Finished .engraver folder initialization.") print("") print_ok_pending("Creating new Ansible playbook. Streaming Ansible output") call(["ansible-galaxy", "init", ansible_dir]) call([ "cp", resource_filename(__name__, "ansible_template/ansible.cfg"), ansible_dir ]) call([ "cp", resource_filename(__name__, "ansible_template/refresh_cache.yml"), ansible_dir ]) print_done("Finished executing Ansible.") print("") print_ok_pending("Updating .gitignore for Engraver files") call("echo '.engraver/clusters/*' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/machines_remove.yml' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/cluster_remove.yml' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/job_submit.yml' >> " + app_name + "/.gitignore", shell=True) call("echo 'ansible/job_kill.yml' >> " + app_name + "/.gitignore", shell=True) print_done("Finished updating .gitignore") print("") print_ok_pending( "Cloning Ansible AWS playbook via Git. Streaming Git output") path = roles_dir + "/aws" call([ "git", "clone", "https://github.com/onyx-platform/engraver-aws.git", path ]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending( "Cloning Ansible Docker playbook via Git. Streaming Git output") path = roles_dir + "/docker" call([ "git", "clone", "https://github.com/onyx-platform/engraver-docker.git", path ]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending( "Cloning Ansible ZooKeeper playbook via Git. Streaming Git output") path = roles_dir + "/zookeeper" call([ "git", "clone", "https://github.com/onyx-platform/engraver-zookeeper.git", path ]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending( "Cloning Ansible BookKeeper playbook via Git. Streaming Git output") path = roles_dir + "/bookkeeper" call([ "git", "clone", "https://github.com/onyx-platform/engraver-bookkeeper.git", path ]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending( "Cloning Ansible Kafka playbook via Git. Streaming Git output") path = roles_dir + "/kafka" call([ "git", "clone", "https://github.com/onyx-platform/engraver-kafka.git", path ]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending( "Cloning Ansible Onyx playbook via Git. Streaming Git output") path = roles_dir + "/onyx" call([ "git", "clone", "https://github.com/onyx-platform/engraver-onyx.git", path ]) drop_git(path) print_done("Finished cloning playbook.") print("") print_ok_pending("Initializing Ansible vars directories") call(["mkdir", "-p", (ansible_dir + "/group_vars")]) call(["mkdir", "-p", (ansible_dir + "/vars/cluster_vars")]) print_done("Finished Ansible vars directory creation.") print("")