def setup():
# creates the env directory if not exists.
create_directory(directory_name='env', include_init=False)
# creates the deployment directory if not exists.
create_directory(directory_name='deployment', include_init=False)
docker_compose_file_content = docker_compose_file
env_file_content = env_file
# creates files
create_file(file_name='docker-compose.yaml',
path='deployment',
content=docker_compose_file_content)
create_file(file_name='env-config.js',
path='deployment',
content=vue_env_config_file)
create_file(file_name='.env', path='env', content=env_file_content)
# edit content of config.py file, set env new file
default_env_filename = ''
config_file_content = default_config_file.replace("<FILENAME>",
default_env_filename)
create_file(file_name='config.py', content=config_file_content)
def _update_env_name(env_name):
try:
with open(os.path.join(os.getcwd(), 'config.py'), 'r') as f:
config_content = f.read()
exec(config_content)
d = eval("settings")
if 'aml' not in d:
d['aml'] = {}
d['aml']['environment'] = env_name
config_file_content = '''settings = ''' + json.dumps(d, indent=2)
create_file(file_name='config.py', content=config_file_content)
except Exception as e:
raise e
def _setup_aml():
# creates the deployment directory if not exists.
create_directory(directory_name='deployment', include_init=False)
# creates the environment directory if not exists.
create_directory(directory_name='env', include_init=False)
create_file(file_name='.env', path='env', content=azureml_env_file)
# sets config.py environment file
default_env_filename = ''
config_file_content = default_config_file.replace("<FILENAME>",
default_env_filename)
create_file(file_name='config.py', content=config_file_content)
for file_name in init_files_aml:
file_options = init_files_aml.get(file_name, file_name)
directory = file_options.get('dir', 'root')
src = os.path.join(get_project_root(), 'mlapp', 'integrations', 'aml',
'generated_files', file_name)
if directory == 'root':
dst = os.path.join(os.getcwd(), file_name)
else:
dst = os.path.join(os.getcwd(), directory, file_name)
# copy file from src to dst
copyfile(src, dst)
# creates amlignore
create_file('.amlignore', content=amlignore_file)
def __rename__(name,
new_name,
dir_name=None,
show_success_msg=True,
delete=False,
force=False):
if name is None or new_name is None:
click.secho(
"ERROR: asset old name and new name are required.\nHint: 'mlapp assets rename OLD_NAME NEW_NAME' is the right command format.",
fg='red')
return True
if name == new_name:
click.secho(
"ERROR: mlapp boilerplate old name and your asset new name are equal. please use a different name or dont use rename option.",
fg='red')
return True
except_list = ["__pycache__"]
if dir_name is None:
old_name_path = os.path.join(os.getcwd(), 'assets', name)
else:
old_name_path = os.path.join(os.getcwd(), 'assets', dir_name)
new_name_path = os.path.join(os.getcwd(), 'assets', new_name)
if not os.path.exists(old_name_path):
click.secho(
f"ERROR: asset \"{name}\" is not exists.\nHint: use 'mlapp assets show' command to view all available assets.",
fg='red')
elif os.path.exists(new_name_path) and not force:
click.escho(f"ERROR: asset \"{new_name}\" is already exists.",
fg='red')
else:
if os.path.exists(new_name_path) and not force:
# shutil.rmtree(new_name_path)
# creates the new model directory
os.mkdir(new_name_path)
# base renaming
if name in rename_dictionary.keys():
model_dict = rename_dictionary.get(name, {})
replace_files = model_dict.keys()
else:
model_dict = rename_dictionary.get('base', {})
replace_files = model_dict.keys()
# rename model name file by file.
for rf in replace_files:
try:
file_additional_path = model_dict[rf].get('inner_path', '')
except_list.append(name + rf)
# files path
if dir_name is None:
full_old_filename = os.path.join(os.getcwd(), "assets",
name,
file_additional_path,
name + rf)
else:
full_old_filename = os.path.join(os.getcwd(), "assets",
dir_name,
file_additional_path,
name + rf)
full_new_filename = os.path.join(os.getcwd(), "assets",
new_name,
file_additional_path,
new_name + rf)
# checks if needs to create sub directories.
if not os.path.exists(
os.path.join(os.getcwd(), "assets", new_name,
file_additional_path)):
os.mkdir(
os.path.join(os.getcwd(), "assets", new_name,
file_additional_path))
words_to_replace = model_dict[rf].get('words', [])
# read file content
file_content = ''''''
with open(full_old_filename, 'r') as file:
lines = file.readlines()
for line in lines:
# line_striped = line.strip()
# replace new file path
for wr_d in words_to_replace:
word = wr_d.get('word', None)
word_type = wr_d.get('word_type', 'append-left')
word_format = wr_d.get('word_format', 'str.lower')
word_pattern = wr_d.get('word_pattern', None)
if word is not None:
# check word format and type
word_format_func = eval(word_format)
old_name_formatted = word_format_func(name)
new_name_formatted = word_format_func(new_name)
if word_type == 'append-left':
if word_pattern is not None and isinstance(
word_pattern, str):
word_pattern = old_name_formatted + word_pattern
elif word_pattern is not None and (
isinstance(word_pattern, list)
or isinstance(
word_pattern, np.array)):
wp_res = ''
for wp in word_pattern:
wp_res += old_name_formatted + wp + '|'
word_pattern = wp_res[:-1]
old_name_formatted = old_name_formatted + word
new_name_formatted = new_name_formatted + word
else:
# 'append-right'
# check for patterns type
if word_pattern is not None and isinstance(
word_pattern, str):
word_pattern += old_name_formatted
elif word_pattern is not None and (
isinstance(word_pattern, list)
or isinstance(
word_pattern, np.array)):
wp_res = ''
for wp in word_pattern:
wp_res += wp + old_name_formatted + '|'
word_pattern = wp_res[:-1]
old_name_formatted = word + old_name_formatted
new_name_formatted = word + new_name_formatted
# checking and replacing model names or using regex
if word_pattern is not None:
line = re.sub(word_pattern,
new_name_formatted, line)
elif old_name_formatted in line:
line = line.replace(
old_name_formatted, new_name_formatted)
file_content += line
# creates the new file with the new name.
create_file(file_name=full_new_filename, content=file_content)
except Exception as e:
shutil.rmtree(new_name_path)
click.secho('ERROR: Oops something bad happened.', fg='red')
return True
# copy the rest of the files
copy_files(old_name_path, new_name_path, except_list)
if delete:
shutil.rmtree(old_name_path)
if show_success_msg:
click.secho('Success: asset "' + name + '" was renamed to "' +
new_name + '".',
fg='green')
def init_command(ml_control_panel, azure_machine_learning, is_gitignore,
is_dockerignore, is_force_init):
if not is_force_init:
is_initiated = False
exsiting_files = os.listdir(os.getcwd())
for f in exsiting_files:
full_path = os.path.join(os.getcwd(), f)
if (os.path.isdir(full_path)
and f in init_files_directories) or '.py' in f:
is_initiated = True
break
if is_initiated:
click.secho(
"ERROR: your project is not empty.\nHint: you can use 'mlapp init --force' option to force init (caution: force may override exsiting files).",
fg='red')
return
# creates the assets directory if not exists.
create_directory(directory_name='assets')
# creates the common directory if not exists.
create_directory(directory_name='common')
# creates the data directory if not exists.
create_directory(directory_name='data', include_init=False)
# generates app file template
app_file_content = app_file
# generates run file template
run_file_content = run_file
# generates utilities file template
utilities_file_content = utilities_file
config_file_content = empty_config_file
# create all files from templates
create_file(file_name='app.py', content=app_file_content)
create_file(file_name='run.py', content=run_file_content)
create_file(file_name='utilities.py',
path='common',
content=utilities_file_content)
create_file(file_name='config.py', content=config_file_content)
create_file(file_name='requirements.txt')
if ml_control_panel:
# creates the env directory if not exists.
create_directory(directory_name='env', include_init=False)
# creates the deployment directory if not exists.
create_directory(directory_name='deployment', include_init=False)
docker_compose_file_content = docker_compose_file
env_file_content = env_file
# creates files
create_file(file_name='docker-compose.yaml',
path='deployment',
content=docker_compose_file_content)
create_file(file_name='.env', path='env', content=env_file_content)
# edit content of config.py file, set env new file
default_env_filename = ""
config_file_content = default_config_file.replace(
"<FILENAME>", default_env_filename)
create_file(file_name='config.py', content=config_file_content)
if azure_machine_learning:
if setup_aml_env is not None:
setup_aml_env()
else:
click.secho(
"Warning: 'azureml sdk is not installed in your environment. please install it and run 'mlapp aml setup' to complete the init operation.",
fg='red')
if not is_gitignore:
if not os.path.exists(os.path.join(os.getcwd(),
'.gitignore')) or is_force_init:
try:
# attempt to get gitignore from github
response = requests.get(
'https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore'
)
if response.status_code != 200:
raise RuntimeError(
'attempted and failed to fetch .gitignore from github.'
' Will use default')
# append ml app gitignore
github_gitignore_file = response.content.decode()
github_gitignore_file += f'\n{gitignore_file}'
# dump
create_file('.gitignore', content=github_gitignore_file)
except:
create_file('.gitignore', content=gitignore_file)
else:
click.secho("Error: '.gitignore' already exists.", fg='red')
if not is_dockerignore:
if not os.path.exists(os.path.join(os.getcwd(),
'.dockerignore')) or is_force_init:
create_file('.dockerignore', content=dockerignore_file)
else:
click.secho("Error: '.dockerignore' already exists.", fg='red')
def create_gitignore(force):
if not os.path.exists(os.path.join(os.getcwd(), '.gitignore')) or force:
create_file('.gitignore', content=gitignore_file)
else:
click.echo("Error: '.gitignore' already exists.")
def create(name, force, with_flow):
# model name validations and transformations.
asset_name = name.lower()
asset_name = asset_name.replace(" ", "_")
asset_name = asset_name.replace("-", "_")
# creates the assets directory if not exists.
create_directory(directory_name='assets')
full_directory_path = os.path.join(os.getcwd(),
os.path.join('assets', asset_name))
if os.path.exists(full_directory_path) and not force:
click.secho(
'Error: ' + asset_name +
' asset already exists.\nHint: please select a unique name to your asset or use --force option to override asset folder.',
fg='red')
return
# create the necessary folders for the new asset, `asset_name` directory and asset configs directory.
create_directory(directory_name=asset_name, path='assets')
create_directory(directory_name='configs',
path=os.path.join('assets', asset_name),
include_init=False)
model_name_capitalized = str_to_camelcase(asset_name)
# generates model manager file template
model_manager_file_content = model_manager_file.format(
model_name_capitalized)
# generates data manager file template
data_manager_file_content = data_manager_file.format(
model_name_capitalized)
if with_flow:
# generates train config file template with flow
train_config_file_content = train_config_file_with_flow.replace(
"<ASSET_NAME>", asset_name)
# generates forecast config file template with flow
forecast_config_file_content = forecast_config_file_with_flow.replace(
"<ASSET_NAME>", asset_name)
else:
# generates train config file template
train_config_file_content = train_config_file.replace(
"<ASSET_NAME>", asset_name)
# generates forecast config file template
forecast_config_file_content = forecast_config_file.replace(
"<ASSET_NAME>", asset_name)
# create all managers templates
create_file(file_name=asset_name + '_model_manager.py',
path=os.path.join('assets', asset_name),
permissions='w+',
content=model_manager_file_content)
create_file(file_name=asset_name + '_data_manager.py',
path=os.path.join('assets', asset_name),
permissions='w+',
content=data_manager_file_content)
# create all configs templates
create_file(file_name=asset_name + '_train_config.json',
path=os.path.join(os.path.join('assets/', asset_name),
'configs'),
permissions='w+',
content=train_config_file_content)
create_file(file_name=asset_name + '_forecast_config.json',
path=os.path.join(os.path.join('assets/', asset_name),
'configs'),
permissions='w+',
content=forecast_config_file_content)
