def setup(): # creates the env directory if not exists. create_directory(directory_name='env', include_init=False) # creates the deployment directory if not exists. create_directory(directory_name='deployment', include_init=False) docker_compose_file_content = docker_compose_file env_file_content = env_file # creates files create_file(file_name='docker-compose.yaml', path='deployment', content=docker_compose_file_content) create_file(file_name='env-config.js', path='deployment', content=vue_env_config_file) create_file(file_name='.env', path='env', content=env_file_content) # edit content of config.py file, set env new file default_env_filename = '' config_file_content = default_config_file.replace("<FILENAME>", default_env_filename) create_file(file_name='config.py', content=config_file_content)
def _update_env_name(env_name): try: with open(os.path.join(os.getcwd(), 'config.py'), 'r') as f: config_content = f.read() exec(config_content) d = eval("settings") if 'aml' not in d: d['aml'] = {} d['aml']['environment'] = env_name config_file_content = '''settings = ''' + json.dumps(d, indent=2) create_file(file_name='config.py', content=config_file_content) except Exception as e: raise e
def _setup_aml(): # creates the deployment directory if not exists. create_directory(directory_name='deployment', include_init=False) # creates the environment directory if not exists. create_directory(directory_name='env', include_init=False) create_file(file_name='.env', path='env', content=azureml_env_file) # sets config.py environment file default_env_filename = '' config_file_content = default_config_file.replace("<FILENAME>", default_env_filename) create_file(file_name='config.py', content=config_file_content) for file_name in init_files_aml: file_options = init_files_aml.get(file_name, file_name) directory = file_options.get('dir', 'root') src = os.path.join(get_project_root(), 'mlapp', 'integrations', 'aml', 'generated_files', file_name) if directory == 'root': dst = os.path.join(os.getcwd(), file_name) else: dst = os.path.join(os.getcwd(), directory, file_name) # copy file from src to dst copyfile(src, dst) # creates amlignore create_file('.amlignore', content=amlignore_file)
def __rename__(name, new_name, dir_name=None, show_success_msg=True, delete=False, force=False): if name is None or new_name is None: click.secho( "ERROR: asset old name and new name are required.\nHint: 'mlapp assets rename OLD_NAME NEW_NAME' is the right command format.", fg='red') return True if name == new_name: click.secho( "ERROR: mlapp boilerplate old name and your asset new name are equal. please use a different name or dont use rename option.", fg='red') return True except_list = ["__pycache__"] if dir_name is None: old_name_path = os.path.join(os.getcwd(), 'assets', name) else: old_name_path = os.path.join(os.getcwd(), 'assets', dir_name) new_name_path = os.path.join(os.getcwd(), 'assets', new_name) if not os.path.exists(old_name_path): click.secho( f"ERROR: asset \"{name}\" is not exists.\nHint: use 'mlapp assets show' command to view all available assets.", fg='red') elif os.path.exists(new_name_path) and not force: click.escho(f"ERROR: asset \"{new_name}\" is already exists.", fg='red') else: if os.path.exists(new_name_path) and not force: # shutil.rmtree(new_name_path) # creates the new model directory os.mkdir(new_name_path) # base renaming if name in rename_dictionary.keys(): model_dict = rename_dictionary.get(name, {}) replace_files = model_dict.keys() else: model_dict = rename_dictionary.get('base', {}) replace_files = model_dict.keys() # rename model name file by file. for rf in replace_files: try: file_additional_path = model_dict[rf].get('inner_path', '') except_list.append(name + rf) # files path if dir_name is None: full_old_filename = os.path.join(os.getcwd(), "assets", name, file_additional_path, name + rf) else: full_old_filename = os.path.join(os.getcwd(), "assets", dir_name, file_additional_path, name + rf) full_new_filename = os.path.join(os.getcwd(), "assets", new_name, file_additional_path, new_name + rf) # checks if needs to create sub directories. if not os.path.exists( os.path.join(os.getcwd(), "assets", new_name, file_additional_path)): os.mkdir( os.path.join(os.getcwd(), "assets", new_name, file_additional_path)) words_to_replace = model_dict[rf].get('words', []) # read file content file_content = '''''' with open(full_old_filename, 'r') as file: lines = file.readlines() for line in lines: # line_striped = line.strip() # replace new file path for wr_d in words_to_replace: word = wr_d.get('word', None) word_type = wr_d.get('word_type', 'append-left') word_format = wr_d.get('word_format', 'str.lower') word_pattern = wr_d.get('word_pattern', None) if word is not None: # check word format and type word_format_func = eval(word_format) old_name_formatted = word_format_func(name) new_name_formatted = word_format_func(new_name) if word_type == 'append-left': if word_pattern is not None and isinstance( word_pattern, str): word_pattern = old_name_formatted + word_pattern elif word_pattern is not None and ( isinstance(word_pattern, list) or isinstance( word_pattern, np.array)): wp_res = '' for wp in word_pattern: wp_res += old_name_formatted + wp + '|' word_pattern = wp_res[:-1] old_name_formatted = old_name_formatted + word new_name_formatted = new_name_formatted + word else: # 'append-right' # check for patterns type if word_pattern is not None and isinstance( word_pattern, str): word_pattern += old_name_formatted elif word_pattern is not None and ( isinstance(word_pattern, list) or isinstance( word_pattern, np.array)): wp_res = '' for wp in word_pattern: wp_res += wp + old_name_formatted + '|' word_pattern = wp_res[:-1] old_name_formatted = word + old_name_formatted new_name_formatted = word + new_name_formatted # checking and replacing model names or using regex if word_pattern is not None: line = re.sub(word_pattern, new_name_formatted, line) elif old_name_formatted in line: line = line.replace( old_name_formatted, new_name_formatted) file_content += line # creates the new file with the new name. create_file(file_name=full_new_filename, content=file_content) except Exception as e: shutil.rmtree(new_name_path) click.secho('ERROR: Oops something bad happened.', fg='red') return True # copy the rest of the files copy_files(old_name_path, new_name_path, except_list) if delete: shutil.rmtree(old_name_path) if show_success_msg: click.secho('Success: asset "' + name + '" was renamed to "' + new_name + '".', fg='green')
def init_command(ml_control_panel, azure_machine_learning, is_gitignore, is_dockerignore, is_force_init): if not is_force_init: is_initiated = False exsiting_files = os.listdir(os.getcwd()) for f in exsiting_files: full_path = os.path.join(os.getcwd(), f) if (os.path.isdir(full_path) and f in init_files_directories) or '.py' in f: is_initiated = True break if is_initiated: click.secho( "ERROR: your project is not empty.\nHint: you can use 'mlapp init --force' option to force init (caution: force may override exsiting files).", fg='red') return # creates the assets directory if not exists. create_directory(directory_name='assets') # creates the common directory if not exists. create_directory(directory_name='common') # creates the data directory if not exists. create_directory(directory_name='data', include_init=False) # generates app file template app_file_content = app_file # generates run file template run_file_content = run_file # generates utilities file template utilities_file_content = utilities_file config_file_content = empty_config_file # create all files from templates create_file(file_name='app.py', content=app_file_content) create_file(file_name='run.py', content=run_file_content) create_file(file_name='utilities.py', path='common', content=utilities_file_content) create_file(file_name='config.py', content=config_file_content) create_file(file_name='requirements.txt') if ml_control_panel: # creates the env directory if not exists. create_directory(directory_name='env', include_init=False) # creates the deployment directory if not exists. create_directory(directory_name='deployment', include_init=False) docker_compose_file_content = docker_compose_file env_file_content = env_file # creates files create_file(file_name='docker-compose.yaml', path='deployment', content=docker_compose_file_content) create_file(file_name='.env', path='env', content=env_file_content) # edit content of config.py file, set env new file default_env_filename = "" config_file_content = default_config_file.replace( "<FILENAME>", default_env_filename) create_file(file_name='config.py', content=config_file_content) if azure_machine_learning: if setup_aml_env is not None: setup_aml_env() else: click.secho( "Warning: 'azureml sdk is not installed in your environment. please install it and run 'mlapp aml setup' to complete the init operation.", fg='red') if not is_gitignore: if not os.path.exists(os.path.join(os.getcwd(), '.gitignore')) or is_force_init: try: # attempt to get gitignore from github response = requests.get( 'https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore' ) if response.status_code != 200: raise RuntimeError( 'attempted and failed to fetch .gitignore from github.' ' Will use default') # append ml app gitignore github_gitignore_file = response.content.decode() github_gitignore_file += f'\n{gitignore_file}' # dump create_file('.gitignore', content=github_gitignore_file) except: create_file('.gitignore', content=gitignore_file) else: click.secho("Error: '.gitignore' already exists.", fg='red') if not is_dockerignore: if not os.path.exists(os.path.join(os.getcwd(), '.dockerignore')) or is_force_init: create_file('.dockerignore', content=dockerignore_file) else: click.secho("Error: '.dockerignore' already exists.", fg='red')
def create_gitignore(force): if not os.path.exists(os.path.join(os.getcwd(), '.gitignore')) or force: create_file('.gitignore', content=gitignore_file) else: click.echo("Error: '.gitignore' already exists.")
def create(name, force, with_flow): # model name validations and transformations. asset_name = name.lower() asset_name = asset_name.replace(" ", "_") asset_name = asset_name.replace("-", "_") # creates the assets directory if not exists. create_directory(directory_name='assets') full_directory_path = os.path.join(os.getcwd(), os.path.join('assets', asset_name)) if os.path.exists(full_directory_path) and not force: click.secho( 'Error: ' + asset_name + ' asset already exists.\nHint: please select a unique name to your asset or use --force option to override asset folder.', fg='red') return # create the necessary folders for the new asset, `asset_name` directory and asset configs directory. create_directory(directory_name=asset_name, path='assets') create_directory(directory_name='configs', path=os.path.join('assets', asset_name), include_init=False) model_name_capitalized = str_to_camelcase(asset_name) # generates model manager file template model_manager_file_content = model_manager_file.format( model_name_capitalized) # generates data manager file template data_manager_file_content = data_manager_file.format( model_name_capitalized) if with_flow: # generates train config file template with flow train_config_file_content = train_config_file_with_flow.replace( "<ASSET_NAME>", asset_name) # generates forecast config file template with flow forecast_config_file_content = forecast_config_file_with_flow.replace( "<ASSET_NAME>", asset_name) else: # generates train config file template train_config_file_content = train_config_file.replace( "<ASSET_NAME>", asset_name) # generates forecast config file template forecast_config_file_content = forecast_config_file.replace( "<ASSET_NAME>", asset_name) # create all managers templates create_file(file_name=asset_name + '_model_manager.py', path=os.path.join('assets', asset_name), permissions='w+', content=model_manager_file_content) create_file(file_name=asset_name + '_data_manager.py', path=os.path.join('assets', asset_name), permissions='w+', content=data_manager_file_content) # create all configs templates create_file(file_name=asset_name + '_train_config.json', path=os.path.join(os.path.join('assets/', asset_name), 'configs'), permissions='w+', content=train_config_file_content) create_file(file_name=asset_name + '_forecast_config.json', path=os.path.join(os.path.join('assets/', asset_name), 'configs'), permissions='w+', content=forecast_config_file_content)