def main(): parser = ArgumentParser() parser.add_argument( "--run", help="Run the autoscaler after wizard finished", action="store_true", default=False, ) args = parser.parse_args() if running_remotely(): hyper_params = AwsAutoScaler.Settings().as_dict() configurations = AwsAutoScaler.Configuration().as_dict() else: print("AWS Autoscaler setup\n") config_file = Path(CONF_FILE).absolute() if config_file.exists() and input_bool( "Load configurations from config file '{}' [Y/n]? ".format( str(CONF_FILE)), default=True, ): with config_file.open("r") as f: conf = yaml.load(f, Loader=yaml.SafeLoader) hyper_params = conf["hyper_params"] configurations = conf["configurations"] else: configurations, hyper_params = run_wizard() try: with config_file.open("w+") as f: conf = { "hyper_params": hyper_params, "configurations": configurations, } yaml.safe_dump(conf, f) except Exception: print( "Error! Could not write configuration file at: {}".format( str(CONF_FILE))) return task = Task.init(project_name="Auto-Scaler", task_name="AWS Auto-Scaler") task.connect(hyper_params) task.connect_configuration(configurations) autoscaler = AwsAutoScaler(hyper_params, configurations) if running_remotely() or args.run: autoscaler.start()
def run_wizard(): # type: () -> Tuple[dict, dict] hyper_params = AwsAutoScaler.Settings() configurations = AwsAutoScaler.Configuration() hyper_params.cloud_credentials_key = get_input("AWS Access Key ID", required=True) hyper_params.cloud_credentials_secret = get_input("AWS Secret Access Key", required=True) hyper_params.cloud_credentials_region = get_input("AWS region name", required=True) # get GIT User/Pass for cloning print( "\nGIT credentials:" "\nEnter GIT username for repository cloning (leave blank for SSH key authentication): [] ", end="", ) git_user = input() if git_user.strip(): print("Enter password for user '{}': ".format(git_user), end="") git_pass = input() print( "Git repository cloning will be using user={} password={}".format( git_user, git_pass)) else: git_user = None git_pass = None hyper_params.git_user = git_user hyper_params.git_pass = git_pass hyper_params.default_docker_image = get_input( "default docker image/parameters", "to use [default is {}]".format(DEFAULT_DOCKER_IMAGE), default=DEFAULT_DOCKER_IMAGE, new_line=True, ) print("\nDefine the type of machines you want the autoscaler to use") resource_configurations = {} while True: resource_name = get_input( "machine type name", "(remember it, we will later use it in the budget section)", required=True, new_line=True, ) resource_configurations[resource_name] = { "instance_type": get_input( "instance type", "for resource '{}' [default is 'g4dn.4xlarge']".format( resource_name), default="g4dn.4xlarge", ), "is_spot": input_bool("is '{}' resource using spot instances? [t/F]".format( resource_name)), "availability_zone": get_input( "availability zone", "for resource '{}' [default is 'us-east-1b']".format( resource_name), default="us-east-1b", ), "ami_id": get_input( "ami_id", "for resource '{}' [default is 'ami-07c95cafbb788face']". format(resource_name), default="ami-07c95cafbb788face", ), "ebs_device_name": get_input( "ebs_device_name", "for resource '{}' [default is '/dev/xvda']".format( resource_name), default="/dev/xvda", ), "ebs_volume_size": input_int( "ebs_volume_size", " for resource '{}' [default is '100']".format(resource_name), default=100, ), "ebs_volume_type": get_input( "ebs_volume_type", "for resource '{}' [default is 'gp2']".format(resource_name), default="gp2", ), } if not input_bool("\nDefine another resource? [y/N]"): break configurations.resource_configurations = resource_configurations configurations.extra_vm_bash_script = input( "\nEnter any pre-execution bash script to be executed on the newly created instances: " ) print("\nSet up the budget\n") queues = defaultdict(list) while True: queue_name = get_input("queue name", required=True) while True: queue_type = get_input( "queue type", "(use the resources names defined earlier)", required=True, ) max_instances = input_int("maximum number of instances allowed", required=True) queues[queue_name].append((queue_type, max_instances)) if not input_bool("\nAdd another type to queue? [y/N]: "): break if not input_bool("Define another queue? [y/N]: "): break configurations.queues = dict(queues) hyper_params.max_idle_time_min = input_int( "maximum idle time", "for the autoscaler (in minutes, default is 15)", default=15, new_line=True, ) hyper_params.polling_interval_time_min = input_int( "polling interval", "for the autoscaler (in minutes, default is 5)", default=5, ) return configurations.as_dict(), hyper_params.as_dict()
def run_wizard(): # type: () -> Tuple[dict, dict] hyper_params = AwsAutoScaler.Settings() configurations = AwsAutoScaler.Configuration() hyper_params.cloud_credentials_key = get_input("AWS Access Key ID", required=True) hyper_params.cloud_credentials_secret = get_input("AWS Secret Access Key", required=True) hyper_params.cloud_credentials_region = get_input("AWS region name", "[us-east-1]", default='us-east-1') # get GIT User/Pass for cloning print( "\nGIT credentials:" "\nEnter GIT username for repository cloning (leave blank for SSH key authentication): [] ", end="", ) git_user = input() if git_user.strip(): print("Enter password for user '{}': ".format(git_user), end="") git_pass = input() print( "Git repository cloning will be using user={} password={}".format( git_user, git_pass)) else: git_user = None git_pass = None hyper_params.git_user = git_user hyper_params.git_pass = git_pass hyper_params.default_docker_image = get_input( "default docker image/parameters", "to use [{}]".format(DEFAULT_DOCKER_IMAGE), default=DEFAULT_DOCKER_IMAGE, new_line=True, ) print("\nConfigure the machine types for the auto-scaler:") print("------------------------------------------------") resource_configurations = {} while True: a_resource = { "instance_type": get_input( "Amazon instance type", "['g4dn.4xlarge']", question='Select', default="g4dn.4xlarge", ), "is_spot": input_bool("Use spot instances? [y/N]"), "availability_zone": get_input( "availability zone", "['us-east-1b']", question='Select', default="us-east-1b", ), "ami_id": get_input( "the Amazon Machine Image id", "['ami-07c95cafbb788face']", question='Select', default="ami-07c95cafbb788face", ), "ebs_device_name": get_input( "the Amazon EBS device", "['/dev/xvda']", default="/dev/xvda", ), "ebs_volume_size": input_int( "the Amazon EBS volume size", "(in GiB) [100]", default=100, ), "ebs_volume_type": get_input( "the Amazon EBS volume type", "['gp2']", default="gp2", ), "key_name": get_input("the Amazon Key Pair name", ), "security_group_ids": input_list("Amazon Security Group ID", ), } while True: resource_name = get_input( "a name for this instance type", "(used in the budget section) For example 'aws4gpu'", question='Select', required=True, ) if resource_name in resource_configurations: print("\tError: instance type '{}' already used!".format( resource_name)) continue break resource_configurations[resource_name] = a_resource if not input_bool("\nDefine another instance type? [y/N]"): break configurations.resource_configurations = resource_configurations configurations.extra_vm_bash_script, num_lines_bash_script = multiline_input( "\nEnter any pre-execution bash script to be executed on the newly created instances []" ) print("Entered {} lines of pre-execution bash script".format( num_lines_bash_script)) configurations.extra_trains_conf, num_lines_trains_conf = multiline_input( "\nEnter anything you'd like to include in your trains.conf file []") print("Entered {} extra lines for trains.conf file".format( num_lines_trains_conf)) print("\nDefine the machines budget:") print("-----------------------------") resource_configurations_names = list( configurations.resource_configurations.keys()) queues = defaultdict(list) while True: while True: queue_name = get_input( "a queue name (for example: 'aws_4gpu_machines')", question='Select', required=True) if queue_name in queues: print("\tError: queue name '{}' already used!".format( queue_name)) continue break while True: valid_instances = [ k for k in resource_configurations_names if k not in (q[0] for q in queues[queue_name]) ] while True: queue_type = get_input( "an instance type to attach to the queue", "{}".format(valid_instances), question="Select", required=True, ) if queue_type not in configurations.resource_configurations: print( "\tError: instance type '{}' not in predefined instances {}!" .format( queue_type, list(configurations.resource_configurations.keys()) )) continue if queue_type in (q[0] for q in queues[queue_name]): print("\tError: instance type '{}' already in {}!".format( queue_type, queue_name)) continue if queue_type in [ q[0] for q in chain.from_iterable(queues.values()) ]: queue_type_new = '{}_{}'.format(queue_type, queue_name) print( "\tInstance type '{}' already used, renaming instance to {}" .format(queue_type, queue_type_new)) configurations.resource_configurations[queue_type_new] = \ dict(**configurations.resource_configurations[queue_type]) queue_type = queue_type_new # make sure the renamed name is not reused if queue_type in (q[0] for q in queues[queue_name]): print("\tError: instance type '{}' already in {}!". format(queue_type, queue_name)) continue break max_instances = input_int( "maximum number of '{}' instances to spin simultaneously (example: 3)" .format(queue_type), required=True) queues[queue_name].append((queue_type, max_instances)) valid_instances = [ k for k in configurations.resource_configurations.keys() if k not in (q[0] for q in queues[queue_name]) ] if not valid_instances: break if not input_bool( "Do you wish to add another instance type to queue? [y/N]: " ): break if not input_bool("\nAdd another queue? [y/N]: "): break configurations.queues = dict(queues) hyper_params.max_idle_time_min = input_int( "maximum idle time", "for the auto-scaler to spin down an instance (in minutes) [15]", default=15, new_line=True, ) hyper_params.polling_interval_time_min = input_int( "instances polling interval", "for the auto-scaler (in minutes) [5]", default=5, ) return configurations.as_dict(), hyper_params.as_dict()
def main(): parser = ArgumentParser() parser.add_argument( "--run", help="Run the autoscaler after wizard finished", action="store_true", default=False, ) parser.add_argument( "--remote", help="Run the autoscaler as a service, launch on the `services` queue", action="store_true", default=False, ) args = parser.parse_args() if running_remotely(): hyper_params = AwsAutoScaler.Settings().as_dict() configurations = AwsAutoScaler.Configuration().as_dict() else: print( "AWS Autoscaler setup wizard\n" "---------------------------\n" "Follow the wizard to configure your AWS auto-scaler service.\n" "Once completed, you will be able to view and change the configuration in the trains-server web UI.\n" "It means there is no need to worry about typos or mistakes :)\n") config_file = Path(CONF_FILE).absolute() if config_file.exists() and input_bool( "Load configurations from config file '{}' [Y/n]? ".format( str(CONF_FILE)), default=True, ): with config_file.open("r") as f: conf = yaml.load(f, Loader=yaml.SafeLoader) hyper_params = conf["hyper_params"] configurations = conf["configurations"] else: configurations, hyper_params = run_wizard() # noinspection PyBroadException try: with config_file.open("w+") as f: conf = { "hyper_params": hyper_params, "configurations": configurations, } yaml.safe_dump(conf, f) except Exception: print( "Error! Could not write configuration file at: {}".format( str(CONF_FILE))) return task = Task.init(project_name="DevOps", task_name="AWS Auto-Scaler", task_type=Task.TaskTypes.service) task.connect(hyper_params) task.connect_configuration(configurations) if args.remote or args.run: print("Running AWS auto-scaler as a service\nExecution log {}".format( task.get_output_log_web_page())) if args.remote: # if we are running remotely enqueue this run, and leave the process # the trains-agent services will pick it up and execute it for us. task.execute_remotely(queue_name='services') autoscaler = AwsAutoScaler(hyper_params, configurations) if running_remotely() or args.run: autoscaler.start()