def delete(self, project_uuid, environment_uuid):
environment_dir = get_environment_directory(
environment_uuid, project_uuid)
shutil.rmtree(environment_dir)
# refresh kernels after change in environments
populate_kernels(app, db)
try:
requests.delete("http://" +
app.config["ORCHEST_API_ADDRESS"] +
"/api/environment-images/%s/%s" %
(project_uuid, environment_uuid))
except Exception as e:
logging.warn("Failed to delete EnvironmentImage: %s" % e)
return jsonify(
{"message": "Environment deletion was successful."})
def put(self, commit_uuid):
commit = Commit.query.filter(
Commit.uuid == commit_uuid).first()
if commit is None:
return '', 404
commit.name = request.json["name"]
commit.tag = name_to_tag(request.json["name"])
commit.base_image = request.json["image_name"]
db.session.commit()
# side effect: update shared kernels directory
populate_kernels(app, db)
return commit_schema.dump(commit)
def post(self, uuid):
if Image.query.filter(
Image.name == request.json["name"]).count() > 0:
raise ImageNameInUse()
new_im = Image(
name=request.json["name"],
language=request.json["language"],
gpu_support=request.json["gpu_support"],
)
db.session.add(new_im)
db.session.commit()
# side effect: update shared kernels directory
populate_kernels(app, db)
return image_schema.dump(new_im)
def post(self, project_uuid, environment_uuid):
# create a new environment in the project
environment_json = request.json.get("environment")
e = Environment(
uuid=str(uuid.uuid4()),
name=environment_json["name"],
project_uuid=project_uuid,
language=environment_json["language"],
setup_script=preprocess_script(
environment_json["setup_script"]),
base_image=environment_json["base_image"],
gpu_support=environment_json["gpu_support"],
)
# use specified uuid if it's not keyword 'new'
if environment_uuid != "new":
e.uuid = environment_uuid
else:
url = (f'http://{app.config["ORCHEST_API_ADDRESS"]}'
f"/api/environments/{project_uuid}")
resp = requests.post(url, json={"uuid": e.uuid})
if resp.status_code != 201:
return {}, resp.status_code, resp.headers.items()
environment_dir = get_environment_directory(
e.uuid, project_uuid)
os.makedirs(environment_dir, exist_ok=True)
serialize_environment_to_disk(e, environment_dir)
# refresh kernels after change in environments
populate_kernels(app, db, project_uuid)
return environment_schema.dump(e)
def create_app():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
socketio = SocketIO(app, cors_allowed_origins="*")
logging.getLogger("engineio").setLevel(logging.ERROR)
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception as e:
logging.warning("Failed to load config.json")
logging.info("Flask CONFIG: %s" % app.config)
db.init_app(app)
# according to SQLAlchemy will only create tables if they do not exist
with app.app_context():
db.create_all()
initialize_default_images(db)
initialize_default_datasources(db, app)
logging.info("Initializing kernels")
populate_kernels(app, db)
# static file serving
@app.route("/public/<path:path>")
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
register_build_views(app, db, socketio)
register_socketio_broadcast(db, socketio)
if ("TELEMETRY_DISABLED" not in app.config
and os.environ.get("FLASK_ENV") != "development"):
# create thread for analytics
scheduler = BackgroundScheduler()
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(
analytics_ping,
"interval",
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app],
)
scheduler.start()
processes = []
if process_start_gate():
file_dir = os.path.dirname(os.path.realpath(__file__))
# TODO: reconsider file permission approach
# file_permission_watcher process
permission_process = Popen(
[
"python3",
"-m",
"scripts.file_permission_watcher",
app.config["USER_DIR"],
],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started file_permission_watcher.py")
processes.append(permission_process)
# docker_builder process
docker_builder_process = Popen(
["python3", "-m", "scripts.docker_builder"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started docker_builder.py")
processes.append(docker_builder_process)
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
def create_app():
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
init_logging()
socketio = SocketIO(app, cors_allowed_origins="*")
if os.getenv("FLASK_ENV") == "development":
app = register_teardown_request(app)
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception:
app.logger.warning("Failed to load config.json")
app.config["ORCHEST_REPO_TAG"] = get_repo_tag()
# create thread for non-cpu bound background tasks, e.g. requests
scheduler = BackgroundScheduler(
job_defaults={
# Infinite amount of grace time, so that if a task cannot be
# instantly executed (e.g. if the webserver is busy) then it
# will eventually be.
"misfire_grace_time": 2**31,
"coalesce": False,
# So that the same job can be in the queue an infinite
# amount of times, e.g. for concurrent requests issuing the
# same tasks.
"max_instances": 2**31,
})
app.config["SCHEDULER"] = scheduler
scheduler.start()
app.logger.info("Flask CONFIG: %s" % app.config)
# Create the database if it does not exist yet. Roughly equal to a
# "CREATE DATABASE IF NOT EXISTS <db_name>" call.
if not database_exists(app.config["SQLALCHEMY_DATABASE_URI"]):
create_database(app.config["SQLALCHEMY_DATABASE_URI"])
db.init_app(app)
ma.init_app(app)
# necessary for migration
Migrate().init_app(app, db)
with app.app_context():
# Alembic does not support calling upgrade() concurrently
if not is_werkzeug_parent():
# Upgrade to the latest revision. This also takes care of
# bringing an "empty" db (no tables) on par.
try:
upgrade()
except Exception as e:
logging.error("Failed to run upgrade() %s [%s]" % (e, type(e)))
# On startup all kernels are freshed. This is because
# updating Orchest might make the kernels in the
# userdir/.orchest/kernels directory invalid.
projs = Project.query.all()
for proj in projs:
try:
populate_kernels(app, db, proj.uuid)
except Exception as e:
logging.error(
"Failed to populate kernels on startup for project %s: %s [%s]"
% (proj.uuid, e, type(e)))
# To avoid multiple removals in case of a flask --reload, so
# that this code runs once per container.
try:
os.mkdir("/tmp/jupyter_lock_removed")
lock_path = os.path.join("/userdir", _config.JUPYTER_USER_CONFIG,
"lab", ".bootlock")
if os.path.exists(lock_path):
app.logger.info("Removing dangling jupyter boot lock.")
os.rmdir(lock_path)
except FileExistsError:
app.logger.info("/tmp/jupyter_lock_removed exists. "
" Not removing the lock again.")
# Telemetry
if not app.config["TELEMETRY_DISABLED"]:
# initialize posthog
posthog.api_key = base64.b64decode(
app.config["POSTHOG_API_KEY"]).decode()
posthog.host = app.config["POSTHOG_HOST"]
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(
analytics_ping,
"interval",
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app],
)
# static file serving
@app.route("/", defaults={"path": ""}, methods=["GET"])
@app.route("/<path:path>", methods=["GET"])
def index(path):
# in Debug mode proxy to CLIENT_DEV_SERVER_URL
if os.environ.get("FLASK_ENV") == "development":
return _proxy(request, app.config["CLIENT_DEV_SERVER_URL"] + "/")
else:
file_path = os.path.join(app.config["STATIC_DIR"], path)
if os.path.isfile(file_path):
return send_from_directory(app.config["STATIC_DIR"], path)
else:
return send_from_directory(app.config["STATIC_DIR"],
"index.html")
register_views(app, db)
register_orchest_api_views(app, db)
register_background_tasks_view(app, db)
register_socketio_broadcast(socketio)
register_analytics_views(app, db)
processes = []
if not is_werkzeug_parent():
file_dir = os.path.dirname(os.path.realpath(__file__))
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
app.logger.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
def create_app():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
socketio = SocketIO(app, cors_allowed_origins="*")
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception as e:
logging.warning("Failed to load config.json")
logging.info("Flask CONFIG: %s" % app.config)
db.init_app(app)
# according to SQLAlchemy will only create tables if they do not exist
with app.app_context():
db.create_all()
initialize_default_images(db)
logging.info("Initializing kernels")
populate_kernels(app, db)
# static file serving
@app.route('/public/<path:path>')
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
register_build_views(app, db, socketio)
if "TELEMETRY_DISABLED" not in app.config:
# create thread for analytics
scheduler = BackgroundScheduler()
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(analytics_ping,
'interval',
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app])
scheduler.start()
# Start threaded file_permission_watcher
# TODO: reconsider file permission approach
# Note: process is never cleaned up, this is permissible because it's only
# executed inside a container.
watcher_file = "/tmp/file_permission_watcher_active"
# guarantee no two python file permission watchers are started
if not os.path.isfile(watcher_file):
with open(watcher_file, "w") as file:
file.write("1")
file_dir = os.path.dirname(os.path.realpath(__file__))
permission_process = Popen([
os.path.join(file_dir, "scripts", "file_permission_watcher.py"),
app.config["USER_DIR"]
])
logging.info("Started file_permission_watcher.py")
return app, socketio
def _collateral(self, project_uuid: str, project_path: str):
"""Create a project on the filesystem.
Given a directory it will detect what parts are missing from the
.orchest directory for the project to be considered initialized,
e.g. the actual .orchest directory, .gitignore file,
environments directory, etc. As part of process initialization
environments are built and kernels refreshed.
Raises:
NotADirectoryError:
FileExistsError:
NotADirectoryError:
"""
full_project_path = os.path.join(current_app.config["PROJECTS_DIR"],
project_path)
# exist_ok=True is there so that this function can be used both
# when initializing a project that was discovered through the
# filesystem or initializing a project from scratch.
os.makedirs(full_project_path, exist_ok=True)
# This would actually be created as a collateral effect when
# populating with default environments, do not rely on that.
expected_internal_dir = os.path.join(full_project_path, ".orchest")
if os.path.isfile(expected_internal_dir):
raise NotADirectoryError(
"The expected internal directory (.orchest) is a file.")
elif not os.path.isdir(expected_internal_dir):
os.makedirs(expected_internal_dir, exist_ok=True)
# Init the .gitignore file if it is not there already.
expected_git_ignore_file = os.path.join(full_project_path, ".orchest",
".gitignore")
if os.path.isdir(expected_git_ignore_file):
raise FileExistsError(".orchest/.gitignore is a directory")
elif not os.path.isfile(expected_git_ignore_file):
with open(expected_git_ignore_file, "w") as ign_file:
ign_file.write(
current_app.config["PROJECT_ORCHEST_GIT_IGNORE_CONTENT"])
# Initialize with default environments only if the project has
# no environments directory.
expected_env_dir = os.path.join(full_project_path, ".orchest",
"environments")
if os.path.isfile(expected_env_dir):
raise NotADirectoryError(
"The expected environments directory (.orchest/environments) "
"is a file.")
elif not os.path.isdir(expected_env_dir):
populate_default_environments(project_uuid)
# Refresh kernels after change in environments, given that
# either we added the default environments or the project has
# environments of its own.
populate_kernels(current_app, db, project_uuid)
# Build environments on project creation.
build_environments_for_project(project_uuid)
Project.query.filter_by(uuid=project_uuid,
path=project_path).update({"status": "READY"})
db.session.commit()
def projects():
projects_dir = os.path.join(app.config["USER_DIR"], "projects")
project_paths = [
name for name in os.listdir(projects_dir)
if os.path.isdir(os.path.join(projects_dir, name))
]
# look for projects that have been removed through the filesystem by the user, cleanup
# dangling resources
fs_removed_projects = Project.query.filter(
Project.path.notin_(project_paths)).all()
for fs_removed_project in fs_removed_projects:
cleanup_project_from_orchest(fs_removed_project)
if len(fs_removed_projects) > 0:
# refresh kernels after change in environments
populate_kernels(app, db)
# detect new projects by detecting directories that were not registered in the db as projects
existing_project_paths = [
project.path for project in Project.query.filter(
Project.path.in_(project_paths)).all()
]
new_project_paths = set(project_paths) - set(existing_project_paths)
for new_project_path in new_project_paths:
try:
init_project(new_project_path)
except Exception as e:
logging.error(
f"Error during project initialization of {new_project_path}: {e}"
)
if request.method == "GET":
projects = projects_schema.dump(Project.query.all())
# Get counts for: pipelines, experiments and environments
for project in projects:
# catch both pipelines of newly initialized projects
# and manually initialized pipelines of existing
# projects
sync_project_pipelines_db_state(project["uuid"])
project["pipeline_count"] = Pipeline.query.filter(
Pipeline.project_uuid == project["uuid"]).count()
project["experiment_count"] = Experiment.query.filter(
Experiment.project_uuid == project["uuid"]).count()
project["environment_count"] = len(
get_environments(project["uuid"]))
return jsonify(projects)
elif request.method == "DELETE":
project_uuid = request.json["project_uuid"]
project = Project.query.filter(
Project.uuid == project_uuid).first()
if project != None:
project_path = project_uuid_to_path(project_uuid)
full_project_path = os.path.join(projects_dir, project_path)
shutil.rmtree(full_project_path)
cleanup_project_from_orchest(project)
# refresh kernels after change in environments
populate_kernels(app, db)
return jsonify({"message": "Project deleted."})
else:
return (
jsonify({
"message":
"Project not found for UUID %s." % project_uuid
}),
404,
)
elif request.method == "POST":
project_path = request.json["name"]
if project_path not in project_paths:
full_project_path = os.path.join(projects_dir, project_path)
if not os.path.isdir(full_project_path):
os.makedirs(full_project_path)
# note that given the current pattern we have in the
# GUI, where we POST and then GET projects,
# this line does not strictly need to be there,
# since the new directory will be picked up
# on the GET request and initialized, placing it
# here is more explicit and less relying
# on the POST->GET pattern from the GUI
try:
init_project(project_path)
except Exception as e:
return (
jsonify({
"message":
"Failed to create the project. Error: %s" % e
}),
500,
)
else:
return (
jsonify(
{"message": "Project directory already exists."}),
409,
)
else:
return (
jsonify({"message": "Project name already exists."}),
409,
)
return jsonify({"message": "Project created."})
def init_project(project_path: str) -> str:
"""Inits an orchest project.
Given a directory it will detect what parts are missing from
the .orchest directory for the project to be considered
initialized, e.g. the actual .orchest directory, .gitignore
file, environments directory, etc.
As part of process initialization environments are
built and kernels refreshed.
Args:
project_path: Directory of the project
Returns:
UUID of the newly initialized project.
"""
projects_dir = os.path.join(app.config["USER_DIR"], "projects")
full_project_path = os.path.join(projects_dir, project_path)
new_project = Project(
uuid=str(uuid.uuid4()),
path=project_path,
)
db.session.add(new_project)
db.session.commit()
try:
# this would actually be created as a collateral effect when populating with default environments,
# let's not rely on that
expected_internal_dir = os.path.join(full_project_path, ".orchest")
if os.path.isfile(expected_internal_dir):
raise NotADirectoryError(
"The expected internal directory (.orchest) is a file.")
elif not os.path.isdir(expected_internal_dir):
os.makedirs(expected_internal_dir)
# init the .gitignore file if it is not there already
expected_git_ignore_file = os.path.join(full_project_path,
".orchest", ".gitignore")
if os.path.isdir(expected_git_ignore_file):
raise FileExistsError(".orchest/.gitignore is a directory")
elif not os.path.isfile(expected_git_ignore_file):
with open(expected_git_ignore_file, "w") as ign_file:
ign_file.write(
app.config["PROJECT_ORCHEST_GIT_IGNORE_CONTENT"])
# initialize with default environments only if the project has no environments directory
expected_env_dir = os.path.join(full_project_path, ".orchest",
"environments")
if os.path.isfile(expected_env_dir):
raise NotADirectoryError(
"The expected environments directory (.orchest/environments) is a file."
)
elif not os.path.isdir(expected_env_dir):
populate_default_environments(new_project.uuid)
# refresh kernels after change in environments, given that either we added the default environments
# or the project has environments of its own
populate_kernels(app, db)
# build environments on project creation
build_environments_for_project(new_project.uuid)
# some calls rely on the project being in the db, like populate_default_environments or populate_kernels,
# for this reason we need to commit the project to the db before the init actually finishes
# if an exception is raised during project init we have to cleanup the newly added project from the db
# TODO: make use of the complete cleanup of a project from orchest once that is implemented, so that we
# use the same code path
except Exception as e:
db.session.delete(new_project)
db.session.commit()
populate_kernels(app, db)
raise e
return new_project.uuid
def create_app(to_migrate_db=False):
"""Create the Flask app and return it.
Args:
to_migrate_db: If True, then only initialize the db.
Returns:
Flask.app
"""
signal.signal(signal.SIGTERM, lambda *args, **kwargs: sys.exit(0))
app = Flask(__name__)
app.config.from_object(config.CONFIG_CLASS)
init_logging()
# In development we want more verbose logging of every request.
if os.getenv("FLASK_ENV") == "development":
app = register_teardown_request(app)
socketio = SocketIO(app, cors_allowed_origins="*")
if not to_migrate_db:
orchest_config = requests.get(
f"http://{config.CONFIG_CLASS.ORCHEST_API_ADDRESS}/api/ctl/orchest-settings"
).json()
app.config.update(orchest_config)
app.config["ORCHEST_REPO_TAG"] = get_repo_tag()
# Create the database if it does not exist yet. Roughly equal to a
# "CREATE DATABASE IF NOT EXISTS <db_name>" call.
if not database_exists(app.config["SQLALCHEMY_DATABASE_URI"]):
create_database(app.config["SQLALCHEMY_DATABASE_URI"])
db.init_app(app)
ma.init_app(app)
# Necessary for DB migrations.
Migrate().init_app(app, db)
# NOTE: In this case we want to return ASAP as otherwise the DB
# might be called (inside this function) before it is migrated.
if to_migrate_db:
return app, None, None
# Add below `to_migrate_db` check, otherwise it will get logged
# twice. Because before the app starts we first migrate.
app.logger.info("Flask CONFIG: %s" % app.config)
# Initialize posthog ASAP, at least before setting up the scheduler
# but after `to_migrate_db`.
if not app.config["TELEMETRY_DISABLED"]:
posthog.api_key = base64.b64decode(
app.config["POSTHOG_API_KEY"]).decode()
posthog.host = app.config["POSTHOG_HOST"]
if not _utils.is_running_from_reloader():
with app.app_context():
try:
if app.config.get("TESTING", False):
# Do nothing.
# In case of tests we always want to run cleanup.
# Because every test will get a clean app, the same
# code should run for all tests.
pass
else:
app.logger.debug(
"Trying to create /tmp/webserver_init_lock")
os.mkdir("/tmp/webserver_init_lock")
app.logger.info(
"/tmp/webserver_init_lock successfully created.")
except FileExistsError:
app.logger.info("/tmp/webserver_init_lock already exists.")
else:
jupyter_boot_lock_path = os.path.join(
"/userdir", _config.JUPYTER_USER_CONFIG, "lab",
".bootlock")
if os.path.exists(jupyter_boot_lock_path):
app.logger.info("Removing dangling jupyter boot lock.")
os.rmdir(jupyter_boot_lock_path)
# On startup all kernels are refreshed. This is because
# updating Orchest might make the kernels in the
# userdir/.orchest/kernels directory invalid.
projs = Project.query.all()
for proj in projs:
try:
populate_kernels(app, db, proj.uuid)
except Exception as e:
logging.error("Failed to populate kernels on startup"
" for project %s: %s [%s]" %
(proj.uuid, e, type(e)))
# create thread for non-cpu bound background tasks, e.g. requests
scheduler = BackgroundScheduler(
job_defaults={
# Infinite amount of grace time, so that if a task cannot be
# instantly executed (e.g. if the webserver is busy) then it
# will eventually be.
"misfire_grace_time": 2**31,
"coalesce": False,
# So that the same job can be in the queue an infinite
# amount of times, e.g. for concurrent requests issuing the
# same tasks.
"max_instances": 2**31,
})
app.config["SCHEDULER"] = scheduler
add_recurring_jobs_to_scheduler(scheduler, app, run_on_add=True)
scheduler.start()
# static file serving
@app.route("/", defaults={"path": ""}, methods=["GET"])
@app.route("/<path:path>", methods=["GET"])
def index(path):
file_path = safe_join(app.config["STATIC_DIR"], path)
if os.path.isfile(file_path):
return send_from_directory(app.config["STATIC_DIR"], path)
else:
return send_from_directory(app.config["STATIC_DIR"],
"index.html",
cache_timeout=0)
register_views(app, db)
register_orchest_api_views(app, db)
register_background_tasks_view(app, db)
register_socketio_broadcast(socketio)
register_analytics_views(app, db)
processes = []
if (os.environ.get("FLASK_ENV") != "development"
or _utils.is_running_from_reloader()):
file_dir = os.path.dirname(os.path.realpath(__file__))
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
app.logger.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
def projects():
project_dir = os.path.join(app.config["USER_DIR"], "projects")
project_paths = [
name for name in os.listdir(project_dir)
if os.path.isdir(os.path.join(project_dir, name))
]
# create UUID entry for all projects that do not yet exist
existing_project_paths = [
project.path for project in Project.query.filter(
Project.path.in_(project_paths)).all()
]
new_project_paths = set(project_paths) - set(existing_project_paths)
for new_project_path in new_project_paths:
new_project = Project(
uuid=str(uuid.uuid4()),
path=new_project_path,
)
db.session.add(new_project)
db.session.commit()
# build environments on project detection
build_environments_for_project(new_project.uuid)
# end of UUID creation
if request.method == "GET":
projects = projects_schema.dump(Project.query.all())
# Get counts for: pipelines, experiments and environments
for project in projects:
project["pipeline_count"] = Pipeline.query.filter(
Pipeline.project_uuid == project["uuid"]).count()
project["experiment_count"] = Experiment.query.filter(
Experiment.project_uuid == project["uuid"]).count()
project["environment_count"] = len(
get_environments(project["uuid"]))
return jsonify(projects)
elif request.method == "DELETE":
project_uuid = request.json["project_uuid"]
project = Project.query.filter(
Project.uuid == project_uuid).first()
if project != None:
project_path = project_uuid_to_path(project_uuid)
full_project_path = os.path.join(project_dir, project_path)
shutil.rmtree(full_project_path)
db.session.delete(project)
db.session.commit()
# refresh kernels after change in environments
populate_kernels(app, db)
return jsonify({"message": "Project deleted."})
else:
return (
jsonify({
"message":
"Project not found for UUID %s." % project_uuid
}),
404,
)
elif request.method == "POST":
project_path = request.json["name"]
if project_path not in project_paths:
full_project_path = os.path.join(project_dir, project_path)
if not os.path.isdir(full_project_path):
new_project = Project(
uuid=str(uuid.uuid4()),
path=project_path,
)
db.session.add(new_project)
db.session.commit()
os.makedirs(full_project_path)
# initialize with default environments
populate_default_environments(new_project.uuid)
# refresh kernels after change in environments
populate_kernels(app, db)
# build environments on project creation
build_environments_for_project(new_project.uuid)
else:
return (
jsonify(
{"message": "Project directory already exists."}),
409,
)
else:
return (
jsonify({"message": "Project name already exists."}),
409,
)
return jsonify({"message": "Project created."})
def _collateral(self, project_uuid: str, project_path: str):
"""Create a project on the filesystem.
Given a directory it will detect what parts are missing from the
.orchest directory for the project to be considered initialized,
e.g. the actual .orchest directory, .gitignore file,
environments directory, etc. As part of process initialization
environments are built and kernels refreshed.
Raises:
NotADirectoryError:
FileExistsError:
NotADirectoryError:
"""
full_project_path = safe_join(current_app.config["PROJECTS_DIR"],
project_path)
# exist_ok=True is there so that this function can be used both
# when initializing a project that was discovered through the
# filesystem or initializing a project from scratch.
os.makedirs(full_project_path, exist_ok=True)
# Create top-level `.gitignore` file with sane defaults, in case
# it not already exists.
root_gitignore = safe_join(full_project_path, ".gitignore")
if not os.path.exists(root_gitignore):
with open(root_gitignore, "w") as f:
f.write("\n".join(
current_app.config["GIT_IGNORE_PROJECT_ROOT"]))
# This would actually be created as a collateral effect when
# populating with default environments, do not rely on that.
expected_internal_dir = safe_join(full_project_path, ".orchest")
if os.path.isfile(expected_internal_dir):
raise NotADirectoryError(
"The expected internal directory (.orchest) is a file.")
elif not os.path.isdir(expected_internal_dir):
os.makedirs(expected_internal_dir, exist_ok=True)
# Init the `.orchest/.gitignore` file if it is not there
# already. NOTE: This `.gitignore` file is created inside the
# `.orchest/` directory because an existing project might be
# added to Orchest and already contain a root-level
# `.gitignore`, which we don't want to inject ourselves in.
expected_git_ignore_file = safe_join(full_project_path, ".orchest",
".gitignore")
if os.path.isdir(expected_git_ignore_file):
raise FileExistsError(".orchest/.gitignore is a directory")
elif not os.path.isfile(expected_git_ignore_file):
with open(expected_git_ignore_file, "w") as ign_file:
ign_file.write("\n".join(
current_app.config["GIT_IGNORE_PROJECT_HIDDEN_ORCHEST"]))
# Initialize with default environments only if the project has
# no environments directory.
expected_env_dir = safe_join(full_project_path, ".orchest",
"environments")
if os.path.isfile(expected_env_dir):
raise NotADirectoryError(
"The expected environments directory (.orchest/environments) "
"is a file.")
elif not os.path.isdir(expected_env_dir):
populate_default_environments(project_uuid)
# Initialize .git directory
expected_git_dir = safe_join(full_project_path, ".git")
# If no git directory exists initialize git repo
if not os.path.exists(expected_git_dir):
p = subprocess.Popen(["git", "init"], cwd=full_project_path)
p.wait()
# Refresh kernels after change in environments, given that
# either we added the default environments or the project has
# environments of its own.
populate_kernels(current_app, db, project_uuid)
resp = requests.post(
f'http://{current_app.config["ORCHEST_API_ADDRESS"]}/api/projects/',
json={
"uuid": project_uuid,
"name": project_path
},
)
if resp.status_code != 201:
raise Exception("Orchest-api project creation failed.")
# Build environments on project creation.
build_environments_for_project(project_uuid)
Project.query.filter_by(uuid=project_uuid,
path=project_path).update({"status": "READY"})
db.session.commit()