def _setup_aml(): # creates the deployment directory if not exists. create_directory(directory_name='deployment', include_init=False) # creates the environment directory if not exists. create_directory(directory_name='env', include_init=False) create_file(file_name='.env', path='env', content=azureml_env_file) # sets config.py environment file default_env_filename = '' config_file_content = default_config_file.replace("<FILENAME>", default_env_filename) create_file(file_name='config.py', content=config_file_content) for file_name in init_files_aml: file_options = init_files_aml.get(file_name, file_name) directory = file_options.get('dir', 'root') src = os.path.join(get_project_root(), 'mlapp', 'integrations', 'aml', 'generated_files', file_name) if directory == 'root': dst = os.path.join(os.getcwd(), file_name) else: dst = os.path.join(os.getcwd(), directory, file_name) # copy file from src to dst copyfile(src, dst) # creates amlignore create_file('.amlignore', content=amlignore_file)
def setup(): # creates the env directory if not exists. create_directory(directory_name='env', include_init=False) # creates the deployment directory if not exists. create_directory(directory_name='deployment', include_init=False) docker_compose_file_content = docker_compose_file env_file_content = env_file # creates files create_file(file_name='docker-compose.yaml', path='deployment', content=docker_compose_file_content) create_file(file_name='env-config.js', path='deployment', content=vue_env_config_file) create_file(file_name='.env', path='env', content=env_file_content) # edit content of config.py file, set env new file default_env_filename = '' config_file_content = default_config_file.replace("<FILENAME>", default_env_filename) create_file(file_name='config.py', content=config_file_content)
def init_command(ml_control_panel, azure_machine_learning, is_gitignore, is_dockerignore, is_force_init): if not is_force_init: is_initiated = False exsiting_files = os.listdir(os.getcwd()) for f in exsiting_files: full_path = os.path.join(os.getcwd(), f) if (os.path.isdir(full_path) and f in init_files_directories) or '.py' in f: is_initiated = True break if is_initiated: click.secho( "ERROR: your project is not empty.\nHint: you can use 'mlapp init --force' option to force init (caution: force may override exsiting files).", fg='red') return # creates the assets directory if not exists. create_directory(directory_name='assets') # creates the common directory if not exists. create_directory(directory_name='common') # creates the data directory if not exists. create_directory(directory_name='data', include_init=False) # generates app file template app_file_content = app_file # generates run file template run_file_content = run_file # generates utilities file template utilities_file_content = utilities_file config_file_content = empty_config_file # create all files from templates create_file(file_name='app.py', content=app_file_content) create_file(file_name='run.py', content=run_file_content) create_file(file_name='utilities.py', path='common', content=utilities_file_content) create_file(file_name='config.py', content=config_file_content) create_file(file_name='requirements.txt') if ml_control_panel: # creates the env directory if not exists. create_directory(directory_name='env', include_init=False) # creates the deployment directory if not exists. create_directory(directory_name='deployment', include_init=False) docker_compose_file_content = docker_compose_file env_file_content = env_file # creates files create_file(file_name='docker-compose.yaml', path='deployment', content=docker_compose_file_content) create_file(file_name='.env', path='env', content=env_file_content) # edit content of config.py file, set env new file default_env_filename = "" config_file_content = default_config_file.replace( "<FILENAME>", default_env_filename) create_file(file_name='config.py', content=config_file_content) if azure_machine_learning: if setup_aml_env is not None: setup_aml_env() else: click.secho( "Warning: 'azureml sdk is not installed in your environment. please install it and run 'mlapp aml setup' to complete the init operation.", fg='red') if not is_gitignore: if not os.path.exists(os.path.join(os.getcwd(), '.gitignore')) or is_force_init: try: # attempt to get gitignore from github response = requests.get( 'https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore' ) if response.status_code != 200: raise RuntimeError( 'attempted and failed to fetch .gitignore from github.' ' Will use default') # append ml app gitignore github_gitignore_file = response.content.decode() github_gitignore_file += f'\n{gitignore_file}' # dump create_file('.gitignore', content=github_gitignore_file) except: create_file('.gitignore', content=gitignore_file) else: click.secho("Error: '.gitignore' already exists.", fg='red') if not is_dockerignore: if not os.path.exists(os.path.join(os.getcwd(), '.dockerignore')) or is_force_init: create_file('.dockerignore', content=dockerignore_file) else: click.secho("Error: '.dockerignore' already exists.", fg='red')
def add(service): # creates the env directory if not exists. create_directory(directory_name='env', include_init=False) # find env file try: env_filename = get_env() except Exception as e: click.secho("ERROR: Please run 'mlapp init' first or set environment file.", fg='red') return env_full_path = os.path.join(os.getcwd(), env_filename) credentials = {} if os.path.exists(env_full_path): service_keys = add_services_options.get(service, False) if service_keys: # get service name from user while True: service_name = validate_str(input("Please name your service (to access the service in the code): ")) is_unique = check_for_service_uniqueness_name(service_name, env_filename) if clean_spaces(service_name) != '' and is_unique: break else: click.secho("ERROR: Service name is required and must be unique, please try again.", fg='red') for key in service_keys: while True: try: body = service_keys[key] new_key = service_name + '_' + key if isinstance(body, str): credentials[new_key] = body else: display_name = body.get('display_name', key) short_description = body.get('short_description', '') # create message to display on the terminal. message = display_name + " (" + short_description + "): " if short_description != '' else key + ": " # get user input user_input = clean_spaces(input(message)) # check for default and required values default_value = body.get('default', None) is_required = body.get('required', False) if default_value is not None and user_input == EMPTY_INPUT: credentials[new_key] = default_value elif user_input == EMPTY_INPUT and is_required: raise Exception("'" + display_name + "' is required, please enter a valid value.") else: # validates user input validations_methods = body.get('validations', []) ans = True for validate in validations_methods: ans &= validate(user_input) if not ans: raise Exception(body.get('error_msg', '')) # transforms user input transformations_methods = body.get('transformations', []) for trans in transformations_methods: user_input = trans(user_input) body_values = body.get('values', False) if body_values: body_values_keys = body['values'].keys() if user_input in body_values_keys: credentials[new_key] = body['values'][user_input] else: raise Exception(body.get('error_msg', 'Oops something bad happened.')) else: credentials[new_key] = user_input break except Exception as e: if e == "": click.secho("Invalid value please try again.", fg='red') else: click.secho(str(e), fg='red') comment = "\n# " + service_name.lower() + " " + service.lower() + " service\n" write_to_file(env_full_path, content=comment + format_service_dictto_txt(credentials)) click.secho( "\nSuccess: " + service.capitalize() + " service was added to your project under the name " + service_name.upper() + ".\n", fg='green') else: click.secho("Service not found.", fg='red') else: # todo create a default env file?. click.secho('Env file not found, please create it before!', fg='red')
def create(name, force, with_flow): # model name validations and transformations. asset_name = name.lower() asset_name = asset_name.replace(" ", "_") asset_name = asset_name.replace("-", "_") # creates the assets directory if not exists. create_directory(directory_name='assets') full_directory_path = os.path.join(os.getcwd(), os.path.join('assets', asset_name)) if os.path.exists(full_directory_path) and not force: click.secho( 'Error: ' + asset_name + ' asset already exists.\nHint: please select a unique name to your asset or use --force option to override asset folder.', fg='red') return # create the necessary folders for the new asset, `asset_name` directory and asset configs directory. create_directory(directory_name=asset_name, path='assets') create_directory(directory_name='configs', path=os.path.join('assets', asset_name), include_init=False) model_name_capitalized = str_to_camelcase(asset_name) # generates model manager file template model_manager_file_content = model_manager_file.format( model_name_capitalized) # generates data manager file template data_manager_file_content = data_manager_file.format( model_name_capitalized) if with_flow: # generates train config file template with flow train_config_file_content = train_config_file_with_flow.replace( "<ASSET_NAME>", asset_name) # generates forecast config file template with flow forecast_config_file_content = forecast_config_file_with_flow.replace( "<ASSET_NAME>", asset_name) else: # generates train config file template train_config_file_content = train_config_file.replace( "<ASSET_NAME>", asset_name) # generates forecast config file template forecast_config_file_content = forecast_config_file.replace( "<ASSET_NAME>", asset_name) # create all managers templates create_file(file_name=asset_name + '_model_manager.py', path=os.path.join('assets', asset_name), permissions='w+', content=model_manager_file_content) create_file(file_name=asset_name + '_data_manager.py', path=os.path.join('assets', asset_name), permissions='w+', content=data_manager_file_content) # create all configs templates create_file(file_name=asset_name + '_train_config.json', path=os.path.join(os.path.join('assets/', asset_name), 'configs'), permissions='w+', content=train_config_file_content) create_file(file_name=asset_name + '_forecast_config.json', path=os.path.join(os.path.join('assets/', asset_name), 'configs'), permissions='w+', content=forecast_config_file_content)