def create_app():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
# read directory mount based config into Flask config
try:
with open("/config/config.json", 'r') as f:
conf_data = json.load(f)
app.config.update(conf_data)
except Exception as e:
logging.warning("Failed to load config.json")
logging.info("Flask CONFIG: %s" % app.config)
db.init_app(app)
# db.create_all()
# static file serving
@app.route('/public/<path:path>')
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
if "TELEMETRY_DISABLED" not in app.config:
# create thread for analytics
scheduler = BackgroundScheduler()
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(analytics_ping, 'interval', minutes=15, args=[app])
scheduler.start()
# Start threaded file_permission_watcher
# TODO: reconsider file permission approach
# Note: process is never cleaned up, this is permissible because it's only
# executed inside a container.
file_dir = os.path.dirname(os.path.realpath(__file__))
permission_process = Popen([
"python3",
os.path.join(file_dir, "scripts/file_permission_watcher.py"),
app.config["USER_DIR"]
])
return app
def create_app():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
socketio = SocketIO(app, cors_allowed_origins="*")
logging.getLogger("engineio").setLevel(logging.ERROR)
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception as e:
logging.warning("Failed to load config.json")
logging.info("Flask CONFIG: %s" % app.config)
db.init_app(app)
# according to SQLAlchemy will only create tables if they do not exist
with app.app_context():
db.create_all()
initialize_default_images(db)
initialize_default_datasources(db, app)
logging.info("Initializing kernels")
populate_kernels(app, db)
# static file serving
@app.route("/public/<path:path>")
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
register_build_views(app, db, socketio)
register_socketio_broadcast(db, socketio)
if ("TELEMETRY_DISABLED" not in app.config
and os.environ.get("FLASK_ENV") != "development"):
# create thread for analytics
scheduler = BackgroundScheduler()
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(
analytics_ping,
"interval",
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app],
)
scheduler.start()
processes = []
if process_start_gate():
file_dir = os.path.dirname(os.path.realpath(__file__))
# TODO: reconsider file permission approach
# file_permission_watcher process
permission_process = Popen(
[
"python3",
"-m",
"scripts.file_permission_watcher",
app.config["USER_DIR"],
],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started file_permission_watcher.py")
processes.append(permission_process)
# docker_builder process
docker_builder_process = Popen(
["python3", "-m", "scripts.docker_builder"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started docker_builder.py")
processes.append(docker_builder_process)
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
def create_app():
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
init_logging()
socketio = SocketIO(app, cors_allowed_origins="*")
if os.getenv("FLASK_ENV") == "development":
app = register_teardown_request(app)
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception:
app.logger.warning("Failed to load config.json")
app.config["ORCHEST_REPO_TAG"] = get_repo_tag()
# create thread for non-cpu bound background tasks, e.g. requests
scheduler = BackgroundScheduler(
job_defaults={
# Infinite amount of grace time, so that if a task cannot be
# instantly executed (e.g. if the webserver is busy) then it
# will eventually be.
"misfire_grace_time": 2**31,
"coalesce": False,
# So that the same job can be in the queue an infinite
# amount of times, e.g. for concurrent requests issuing the
# same tasks.
"max_instances": 2**31,
})
app.config["SCHEDULER"] = scheduler
scheduler.start()
app.logger.info("Flask CONFIG: %s" % app.config)
# Create the database if it does not exist yet. Roughly equal to a
# "CREATE DATABASE IF NOT EXISTS <db_name>" call.
if not database_exists(app.config["SQLALCHEMY_DATABASE_URI"]):
create_database(app.config["SQLALCHEMY_DATABASE_URI"])
db.init_app(app)
ma.init_app(app)
# necessary for migration
Migrate().init_app(app, db)
with app.app_context():
# Alembic does not support calling upgrade() concurrently
if not is_werkzeug_parent():
# Upgrade to the latest revision. This also takes care of
# bringing an "empty" db (no tables) on par.
try:
upgrade()
except Exception as e:
logging.error("Failed to run upgrade() %s [%s]" % (e, type(e)))
# On startup all kernels are freshed. This is because
# updating Orchest might make the kernels in the
# userdir/.orchest/kernels directory invalid.
projs = Project.query.all()
for proj in projs:
try:
populate_kernels(app, db, proj.uuid)
except Exception as e:
logging.error(
"Failed to populate kernels on startup for project %s: %s [%s]"
% (proj.uuid, e, type(e)))
# To avoid multiple removals in case of a flask --reload, so
# that this code runs once per container.
try:
os.mkdir("/tmp/jupyter_lock_removed")
lock_path = os.path.join("/userdir", _config.JUPYTER_USER_CONFIG,
"lab", ".bootlock")
if os.path.exists(lock_path):
app.logger.info("Removing dangling jupyter boot lock.")
os.rmdir(lock_path)
except FileExistsError:
app.logger.info("/tmp/jupyter_lock_removed exists. "
" Not removing the lock again.")
# Telemetry
if not app.config["TELEMETRY_DISABLED"]:
# initialize posthog
posthog.api_key = base64.b64decode(
app.config["POSTHOG_API_KEY"]).decode()
posthog.host = app.config["POSTHOG_HOST"]
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(
analytics_ping,
"interval",
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app],
)
# static file serving
@app.route("/", defaults={"path": ""}, methods=["GET"])
@app.route("/<path:path>", methods=["GET"])
def index(path):
# in Debug mode proxy to CLIENT_DEV_SERVER_URL
if os.environ.get("FLASK_ENV") == "development":
return _proxy(request, app.config["CLIENT_DEV_SERVER_URL"] + "/")
else:
file_path = os.path.join(app.config["STATIC_DIR"], path)
if os.path.isfile(file_path):
return send_from_directory(app.config["STATIC_DIR"], path)
else:
return send_from_directory(app.config["STATIC_DIR"],
"index.html")
register_views(app, db)
register_orchest_api_views(app, db)
register_background_tasks_view(app, db)
register_socketio_broadcast(socketio)
register_analytics_views(app, db)
processes = []
if not is_werkzeug_parent():
file_dir = os.path.dirname(os.path.realpath(__file__))
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
app.logger.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
def create_app():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
socketio = SocketIO(app, cors_allowed_origins="*")
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception as e:
logging.warning("Failed to load config.json")
logging.info("Flask CONFIG: %s" % app.config)
db.init_app(app)
# according to SQLAlchemy will only create tables if they do not exist
with app.app_context():
db.create_all()
initialize_default_images(db)
logging.info("Initializing kernels")
populate_kernels(app, db)
# static file serving
@app.route('/public/<path:path>')
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
register_build_views(app, db, socketio)
if "TELEMETRY_DISABLED" not in app.config:
# create thread for analytics
scheduler = BackgroundScheduler()
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(analytics_ping,
'interval',
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app])
scheduler.start()
# Start threaded file_permission_watcher
# TODO: reconsider file permission approach
# Note: process is never cleaned up, this is permissible because it's only
# executed inside a container.
watcher_file = "/tmp/file_permission_watcher_active"
# guarantee no two python file permission watchers are started
if not os.path.isfile(watcher_file):
with open(watcher_file, "w") as file:
file.write("1")
file_dir = os.path.dirname(os.path.realpath(__file__))
permission_process = Popen([
os.path.join(file_dir, "scripts", "file_permission_watcher.py"),
app.config["USER_DIR"]
])
logging.info("Started file_permission_watcher.py")
return app, socketio
def create_app():
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
socketio = SocketIO(app, cors_allowed_origins="*")
logging.getLogger("engineio").setLevel(logging.ERROR)
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception as e:
logging.warning("Failed to load config.json")
app.config["ORCHEST_REPO_TAG"] = get_repo_tag()
# create thread for non-cpu bound background tasks, e.g.
# requests
scheduler = BackgroundScheduler()
app.config["SCHEDULER"] = scheduler
logging.info("Flask CONFIG: %s" % app.config)
db.init_app(app)
ma.init_app(app)
# according to SQLAlchemy will only create tables if they do not exist
with app.app_context():
db.create_all()
# this class is only serialized on disk
# see the Environment model
# to avoid errors like querying the Environments table
# to see if there is any environemnt (and getting
# no environments while they might actually be there), the
# table is deleted, so that it will produce a ~loud~ error
Environment.__table__.drop(db.engine)
initialize_default_datasources(db, app)
# Telemetry
if not app.config["TELEMETRY_DISABLED"]:
# initialize posthog
posthog.api_key = base64.b64decode(app.config["POSTHOG_API_KEY"]).decode()
posthog.host = app.config["POSTHOG_HOST"]
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(
analytics_ping,
"interval",
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app],
)
scheduler.start()
# static file serving
@app.route("/public/<path:path>")
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
register_orchest_api_views(app, db)
register_background_tasks_view(app, db)
register_socketio_broadcast(db, socketio)
register_analytics_views(app, db)
processes = []
if process_start_gate():
file_dir = os.path.dirname(os.path.realpath(__file__))
# TODO: reconsider file permission approach
# file_permission_watcher process
permission_process = Popen(
[
"python3",
"-m",
"scripts.file_permission_watcher",
app.config["USER_DIR"],
],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started file_permission_watcher.py")
processes.append(permission_process)
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
logging.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
def create_app():
app = Flask(__name__)
app.config.from_object(CONFIG_CLASS)
init_logging()
socketio = SocketIO(app, cors_allowed_origins="*")
if os.getenv("FLASK_ENV") == "development":
app = register_teardown_request(app)
# read directory mount based config into Flask config
try:
conf_data = get_user_conf()
app.config.update(conf_data)
except Exception:
app.logger.warning("Failed to load config.json")
app.config["ORCHEST_REPO_TAG"] = get_repo_tag()
# create thread for non-cpu bound background tasks, e.g. requests
scheduler = BackgroundScheduler(
job_defaults={
# Infinite amount of grace time, so that if a task cannot be
# instantly executed (e.g. if the webserver is busy) then it
# will eventually be.
"misfire_grace_time": 2**31,
"coalesce": False,
# So that the same job can be in the queue an infinite
# amount of times, e.g. for concurrent requests issuing the
# same tasks.
"max_instances": 2**31,
})
app.config["SCHEDULER"] = scheduler
scheduler.start()
app.logger.info("Flask CONFIG: %s" % app.config)
# Create the database if it does not exist yet. Roughly equal to a
# "CREATE DATABASE IF NOT EXISTS <db_name>" call.
if not database_exists(app.config["SQLALCHEMY_DATABASE_URI"]):
create_database(app.config["SQLALCHEMY_DATABASE_URI"])
db.init_app(app)
ma.init_app(app)
# necessary for migration
Migrate().init_app(app, db)
with app.app_context():
# Alembic does not support calling upgrade() concurrently
if not is_werkzeug_parent():
# Upgrade to the latest revision. This also takes care of
# bringing an "empty" db (no tables) on par.
try:
upgrade()
except Exception as e:
logging.error("Failed to run upgrade() %s [%s]" % (e, type(e)))
initialize_default_datasources(db, app)
# Telemetry
if not app.config["TELEMETRY_DISABLED"]:
# initialize posthog
posthog.api_key = base64.b64decode(
app.config["POSTHOG_API_KEY"]).decode()
posthog.host = app.config["POSTHOG_HOST"]
# send a ping now
analytics_ping(app)
# and every 15 minutes
scheduler.add_job(
analytics_ping,
"interval",
minutes=app.config["TELEMETRY_INTERVAL"],
args=[app],
)
# static file serving
@app.route("/public/<path:path>")
def send_files(path):
return send_from_directory("../static", path)
register_views(app, db)
register_orchest_api_views(app, db)
register_background_tasks_view(app, db)
register_socketio_broadcast(db, socketio)
register_analytics_views(app, db)
processes = []
if not is_werkzeug_parent():
file_dir = os.path.dirname(os.path.realpath(__file__))
# log_streamer process
log_streamer_process = Popen(
["python3", "-m", "scripts.log_streamer"],
cwd=os.path.join(file_dir, ".."),
stderr=subprocess.STDOUT,
)
app.logger.info("Started log_streamer.py")
processes.append(log_streamer_process)
return app, socketio, processes
