def gitflix(project, setup_test_models):
# Compile the test files and make sure that the proper m5oc file was generated
ProjectCompiler(project).compile()
# Load the m5oc file for gitflix
m5oc_file = project.run_dir("models", "custom", "gitflix.topic.m5oc")
return M5ocFile.load(m5oc_file)
def install():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
compiler = ProjectCompiler(project)
install_service = PluginInstallService(project)
config_service = ConfigService(project)
plugin = config_service.find_plugin(plugin_name, plugin_type=plugin_type)
run_venv = install_service.create_venv(plugin)
run_install_plugin = install_service.install_plugin(plugin)
if plugin_type is PluginType.MODELS:
try:
compiler.compile()
except Exception as e:
pass
return jsonify(plugin.canonical())
def model(ctx, project):
compiler = ctx.obj["compiler"] = ProjectCompiler(project)
compiler.parse()
if ctx.invoked_subcommand is None:
click.secho("Local", fg="green")
for model in compiler.topics:
click.echo(f'{model["namespace"]}/{model["name"]}')
print()
click.secho("Packaged", fg="green")
for model in compiler.package_topics:
click.echo(f'{model["namespace"]}/{model["name"]}')
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
discovery = PluginDiscoveryService(project)
target_plugin = discovery.find_plugin(plugin_type, plugin_name)
config_service = ConfigService(project)
add_service = ProjectAddService(project)
install_service = PluginInstallService(project)
ignored_types = [target_plugin.type, PluginType.TRANSFORMS]
has_model = False
batched = []
for plugin in discovery.plugins():
if plugin.namespace == target_plugin.namespace:
if plugin.type not in ignored_types:
add_service.add(plugin.type, plugin.name)
plugin_install = config_service.find_plugin(
plugin.name, plugin_type=plugin.type)
batched.append(plugin_install.canonical())
run_venv = install_service.create_venv(plugin_install)
run_install_plugin = install_service.install_plugin(
plugin_install)
if plugin.type is PluginType.MODELS:
has_model = True
if has_model:
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
return jsonify(batched)
def lint_all(compile):
project = Project.find()
compiler = ProjectCompiler(project)
try:
compiler.parse()
except MeltanoAnalysisFileParserError as e:
return jsonify(
{
"result": False,
"errors": [{"message": e.message, "file_name": e.file_name}],
}
)
if compile:
compiler.compile()
return jsonify({"result": True})
def compile_models(self):
click.secho("Recompiling models...", fg="blue")
ProjectCompiler(self.project).compile()
def __init__(self, project: Project, compiler: ProjectCompiler = None):
self.project = project
self.compiler = compiler or ProjectCompiler(project)
self.observer = self.setup_observer()
def compile_models(self):
# Make sure we load the _new_ Meltano version's ProjectCompiler
importlib.reload(meltano.core.compiler.project_compiler)
from meltano.core.compiler.project_compiler import ProjectCompiler
ProjectCompiler(self.project).compile()
def create_app(config={}):
project = Project.find()
app = Flask(__name__,
instance_path=str(project.root),
instance_relative_config=True)
app.config.from_object("meltano.api.config")
app.config.from_pyfile("ui.cfg", silent=True)
app.config.update(**config)
# register
project_engine(project,
engine_uri=app.config["SQLALCHEMY_DATABASE_URI"],
default=True)
# Initial compilation
compiler = ProjectCompiler(project)
try:
compiler.compile()
except Exception as e:
pass
# Logging
file_handler = logging.handlers.RotatingFileHandler(str(
project.run_dir("meltano-ui.log")),
backupCount=3)
stdout_handler = logging.StreamHandler()
logger.setLevel(logging.DEBUG)
logger.addHandler(file_handler)
logger.addHandler(stdout_handler)
# 1) Extensions
security_options = {}
from .models import db
from .mail import mail
from .executor import setup_executor
from .security import security, users, setup_security
from .security.oauth import setup_oauth
from .json import setup_json
db.init_app(app)
mail.init_app(app)
setup_executor(app, project)
setup_security(app, project)
setup_oauth(app)
setup_json(app)
CORS(app, origins="*")
# 2) Register the URL Converters
from .url_converters import PluginRefConverter
app.url_map.converters["plugin_ref"] = PluginRefConverter
# 3) Register the controllers
from .controllers.root import root
from .controllers.dashboards import dashboardsBP
from .controllers.reports import reportsBP
from .controllers.repos import reposBP
from .controllers.settings import settingsBP
from .controllers.sql import sqlBP
from .controllers.orchestrations import orchestrationsBP
from .controllers.plugins import pluginsBP
app.register_blueprint(root)
app.register_blueprint(dashboardsBP)
app.register_blueprint(reportsBP)
app.register_blueprint(reposBP)
app.register_blueprint(settingsBP)
app.register_blueprint(sqlBP)
app.register_blueprint(orchestrationsBP)
app.register_blueprint(pluginsBP)
if app.config["PROFILE"]:
from .profiler import init
init(app)
# Google Analytics setup
tracker = GoogleAnalyticsTracker(project)
@app.before_request
def setup_js_context():
# setup the appUrl
appUrl = urlsplit(request.host_url)
g.jsContext = {"appUrl": appUrl.geturl()[:-1]}
if tracker.send_anonymous_usage_stats:
g.jsContext["isSendAnonymousUsageStats"] = True
g.jsContext["projectId"] = tracker.project_id
g.jsContext["version"] = meltano.__version__
# setup the airflowUrl
try:
airflow = ConfigService(project).find_plugin("airflow")
settings = PluginSettingsService(project)
airflow_port, _ = settings.get_value(db.session, airflow,
"webserver.web_server_port")
g.jsContext["airflowUrl"] = appUrl._replace(
netloc=f"{appUrl.hostname}:{airflow_port}").geturl()[:-1]
except (PluginMissingError, PluginSettingMissingError):
pass
# setup the dbtDocsUrl
g.jsContext["dbtDocsUrl"] = appUrl._replace(
path="/-/dbt/").geturl()[:-1]
@app.after_request
def after_request(res):
request_message = f"[{request.url}]"
if request.method != "OPTIONS":
request_message += f" as {current_user}"
logger.info(request_message)
res.headers["X-Meltano-Version"] = meltano.__version__
return res
return app
def project_compiler(project):
return ProjectCompiler(project)