def _setup_aml():
# creates the deployment directory if not exists.
create_directory(directory_name='deployment', include_init=False)
# creates the environment directory if not exists.
create_directory(directory_name='env', include_init=False)
create_file(file_name='.env', path='env', content=azureml_env_file)
# sets config.py environment file
default_env_filename = ''
config_file_content = default_config_file.replace("<FILENAME>",
default_env_filename)
create_file(file_name='config.py', content=config_file_content)
for file_name in init_files_aml:
file_options = init_files_aml.get(file_name, file_name)
directory = file_options.get('dir', 'root')
src = os.path.join(get_project_root(), 'mlapp', 'integrations', 'aml',
'generated_files', file_name)
if directory == 'root':
dst = os.path.join(os.getcwd(), file_name)
else:
dst = os.path.join(os.getcwd(), directory, file_name)
# copy file from src to dst
copyfile(src, dst)
# creates amlignore
create_file('.amlignore', content=amlignore_file)
def setup():
# creates the env directory if not exists.
create_directory(directory_name='env', include_init=False)
# creates the deployment directory if not exists.
create_directory(directory_name='deployment', include_init=False)
docker_compose_file_content = docker_compose_file
env_file_content = env_file
# creates files
create_file(file_name='docker-compose.yaml',
path='deployment',
content=docker_compose_file_content)
create_file(file_name='env-config.js',
path='deployment',
content=vue_env_config_file)
create_file(file_name='.env', path='env', content=env_file_content)
# edit content of config.py file, set env new file
default_env_filename = ''
config_file_content = default_config_file.replace("<FILENAME>",
default_env_filename)
create_file(file_name='config.py', content=config_file_content)
def init_command(ml_control_panel, azure_machine_learning, is_gitignore,
is_dockerignore, is_force_init):
if not is_force_init:
is_initiated = False
exsiting_files = os.listdir(os.getcwd())
for f in exsiting_files:
full_path = os.path.join(os.getcwd(), f)
if (os.path.isdir(full_path)
and f in init_files_directories) or '.py' in f:
is_initiated = True
break
if is_initiated:
click.secho(
"ERROR: your project is not empty.\nHint: you can use 'mlapp init --force' option to force init (caution: force may override exsiting files).",
fg='red')
return
# creates the assets directory if not exists.
create_directory(directory_name='assets')
# creates the common directory if not exists.
create_directory(directory_name='common')
# creates the data directory if not exists.
create_directory(directory_name='data', include_init=False)
# generates app file template
app_file_content = app_file
# generates run file template
run_file_content = run_file
# generates utilities file template
utilities_file_content = utilities_file
config_file_content = empty_config_file
# create all files from templates
create_file(file_name='app.py', content=app_file_content)
create_file(file_name='run.py', content=run_file_content)
create_file(file_name='utilities.py',
path='common',
content=utilities_file_content)
create_file(file_name='config.py', content=config_file_content)
create_file(file_name='requirements.txt')
if ml_control_panel:
# creates the env directory if not exists.
create_directory(directory_name='env', include_init=False)
# creates the deployment directory if not exists.
create_directory(directory_name='deployment', include_init=False)
docker_compose_file_content = docker_compose_file
env_file_content = env_file
# creates files
create_file(file_name='docker-compose.yaml',
path='deployment',
content=docker_compose_file_content)
create_file(file_name='.env', path='env', content=env_file_content)
# edit content of config.py file, set env new file
default_env_filename = ""
config_file_content = default_config_file.replace(
"<FILENAME>", default_env_filename)
create_file(file_name='config.py', content=config_file_content)
if azure_machine_learning:
if setup_aml_env is not None:
setup_aml_env()
else:
click.secho(
"Warning: 'azureml sdk is not installed in your environment. please install it and run 'mlapp aml setup' to complete the init operation.",
fg='red')
if not is_gitignore:
if not os.path.exists(os.path.join(os.getcwd(),
'.gitignore')) or is_force_init:
try:
# attempt to get gitignore from github
response = requests.get(
'https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore'
)
if response.status_code != 200:
raise RuntimeError(
'attempted and failed to fetch .gitignore from github.'
' Will use default')
# append ml app gitignore
github_gitignore_file = response.content.decode()
github_gitignore_file += f'\n{gitignore_file}'
# dump
create_file('.gitignore', content=github_gitignore_file)
except:
create_file('.gitignore', content=gitignore_file)
else:
click.secho("Error: '.gitignore' already exists.", fg='red')
if not is_dockerignore:
if not os.path.exists(os.path.join(os.getcwd(),
'.dockerignore')) or is_force_init:
create_file('.dockerignore', content=dockerignore_file)
else:
click.secho("Error: '.dockerignore' already exists.", fg='red')
def add(service):
# creates the env directory if not exists.
create_directory(directory_name='env', include_init=False)
# find env file
try:
env_filename = get_env()
except Exception as e:
click.secho("ERROR: Please run 'mlapp init' first or set environment file.", fg='red')
return
env_full_path = os.path.join(os.getcwd(), env_filename)
credentials = {}
if os.path.exists(env_full_path):
service_keys = add_services_options.get(service, False)
if service_keys:
# get service name from user
while True:
service_name = validate_str(input("Please name your service (to access the service in the code): "))
is_unique = check_for_service_uniqueness_name(service_name, env_filename)
if clean_spaces(service_name) != '' and is_unique:
break
else:
click.secho("ERROR: Service name is required and must be unique, please try again.", fg='red')
for key in service_keys:
while True:
try:
body = service_keys[key]
new_key = service_name + '_' + key
if isinstance(body, str):
credentials[new_key] = body
else:
display_name = body.get('display_name', key)
short_description = body.get('short_description', '')
# create message to display on the terminal.
message = display_name + " (" + short_description + "): " if short_description != '' else key + ": "
# get user input
user_input = clean_spaces(input(message))
# check for default and required values
default_value = body.get('default', None)
is_required = body.get('required', False)
if default_value is not None and user_input == EMPTY_INPUT:
credentials[new_key] = default_value
elif user_input == EMPTY_INPUT and is_required:
raise Exception("'" + display_name + "' is required, please enter a valid value.")
else:
# validates user input
validations_methods = body.get('validations', [])
ans = True
for validate in validations_methods:
ans &= validate(user_input)
if not ans:
raise Exception(body.get('error_msg', ''))
# transforms user input
transformations_methods = body.get('transformations', [])
for trans in transformations_methods:
user_input = trans(user_input)
body_values = body.get('values', False)
if body_values:
body_values_keys = body['values'].keys()
if user_input in body_values_keys:
credentials[new_key] = body['values'][user_input]
else:
raise Exception(body.get('error_msg', 'Oops something bad happened.'))
else:
credentials[new_key] = user_input
break
except Exception as e:
if e == "":
click.secho("Invalid value please try again.", fg='red')
else:
click.secho(str(e), fg='red')
comment = "\n# " + service_name.lower() + " " + service.lower() + " service\n"
write_to_file(env_full_path, content=comment + format_service_dictto_txt(credentials))
click.secho(
"\nSuccess: " + service.capitalize() + " service was added to your project under the name " + service_name.upper() + ".\n",
fg='green')
else:
click.secho("Service not found.", fg='red')
else:
# todo create a default env file?.
click.secho('Env file not found, please create it before!', fg='red')
def create(name, force, with_flow):
# model name validations and transformations.
asset_name = name.lower()
asset_name = asset_name.replace(" ", "_")
asset_name = asset_name.replace("-", "_")
# creates the assets directory if not exists.
create_directory(directory_name='assets')
full_directory_path = os.path.join(os.getcwd(),
os.path.join('assets', asset_name))
if os.path.exists(full_directory_path) and not force:
click.secho(
'Error: ' + asset_name +
' asset already exists.\nHint: please select a unique name to your asset or use --force option to override asset folder.',
fg='red')
return
# create the necessary folders for the new asset, `asset_name` directory and asset configs directory.
create_directory(directory_name=asset_name, path='assets')
create_directory(directory_name='configs',
path=os.path.join('assets', asset_name),
include_init=False)
model_name_capitalized = str_to_camelcase(asset_name)
# generates model manager file template
model_manager_file_content = model_manager_file.format(
model_name_capitalized)
# generates data manager file template
data_manager_file_content = data_manager_file.format(
model_name_capitalized)
if with_flow:
# generates train config file template with flow
train_config_file_content = train_config_file_with_flow.replace(
"<ASSET_NAME>", asset_name)
# generates forecast config file template with flow
forecast_config_file_content = forecast_config_file_with_flow.replace(
"<ASSET_NAME>", asset_name)
else:
# generates train config file template
train_config_file_content = train_config_file.replace(
"<ASSET_NAME>", asset_name)
# generates forecast config file template
forecast_config_file_content = forecast_config_file.replace(
"<ASSET_NAME>", asset_name)
# create all managers templates
create_file(file_name=asset_name + '_model_manager.py',
path=os.path.join('assets', asset_name),
permissions='w+',
content=model_manager_file_content)
create_file(file_name=asset_name + '_data_manager.py',
path=os.path.join('assets', asset_name),
permissions='w+',
content=data_manager_file_content)
# create all configs templates
create_file(file_name=asset_name + '_train_config.json',
path=os.path.join(os.path.join('assets/', asset_name),
'configs'),
permissions='w+',
content=train_config_file_content)
create_file(file_name=asset_name + '_forecast_config.json',
path=os.path.join(os.path.join('assets/', asset_name),
'configs'),
permissions='w+',
content=forecast_config_file_content)
