def __init__(self, elt_context: ELTContext):
self.context = elt_context
self.project_settings_service = ProjectSettingsService(
self.context.project,
config_service=self.context.plugins_service.config_service,
)
class UIAvailableWorker(threading.Thread):
def __init__(self, project, open_browser=False):
super().__init__()
self.project = project
self.open_browser = open_browser
self.settings_service = ProjectSettingsService(self.project)
self._terminate = False
def run(self):
bind_port = self.settings_service.get("ui.bind_port")
url = f"http://localhost:{bind_port}"
server_name = self.settings_service.get("ui.server_name")
headers = {"Host": server_name}
while not self._terminate:
try:
response = requests.get(url, headers=headers)
if response.status_code == 200:
click.secho(f"Meltano UI is now available at {url}",
fg="green")
if self.open_browser:
webbrowser.open(url)
self._terminate = True
except:
pass
time.sleep(2)
def stop(self):
self._terminate = True
def __init__(
self,
project: Project,
plugin: ProjectPlugin,
*args,
plugins_service: ProjectPluginsService = None,
**kwargs,
):
super().__init__(project, *args, **kwargs)
self.plugin = plugin
self.plugins_service = plugins_service or ProjectPluginsService(self.project)
project_settings_service = ProjectSettingsService(
self.project, config_service=self.plugins_service.config_service
)
self.env_override = {
**project_settings_service.env,
**project_settings_service.as_env(),
**self.env_override,
**self.plugin.info_env,
}
self._inherited_settings_service = None
def test_ui_setup(self, project, cli_runner, monkeypatch):
monkeypatch.setenv("MELTANO_UI_SECRET_KEY", "existing_secret_key")
with mock.patch("meltano.cli.ui.secrets.token_hex",
return_value="fake_secret"):
result = cli_runner.invoke(cli,
["ui", "setup", "meltano.example.com"])
assert_cli_runner(result)
assert (
"Setting 'ui.secret_key' has already been set in the environment"
in result.stdout)
settings_service = ProjectSettingsService(project)
assert settings_service.get_with_source("ui.server_name") == (
"meltano.example.com",
SettingValueStore.DOTENV,
)
assert settings_service.get_with_source("ui.secret_key") == (
"existing_secret_key",
SettingValueStore.ENV,
)
assert settings_service.get_with_source("ui.password_salt") == (
"fake_secret",
SettingValueStore.DOTENV,
)
def decorated(*args, **kwargs):
project = Project.find()
settings_service = ProjectSettingsService(project)
if settings_service.get("ui.readonly"):
return (
jsonify({
"error": True,
"code": "Meltano UI is running in read-only mode"
}),
HTTP_READONLY_CODE,
)
if settings_service.get(
"ui.anonymous_readonly") and current_user.is_anonymous:
return (
jsonify({
"error":
True,
"code":
"Meltano UI is running in read-only mode until you sign in",
}),
HTTP_READONLY_CODE,
)
return f(*args, **kwargs)
def __init__(self, project, open_browser=False):
super().__init__()
self.project = project
self.open_browser = open_browser
self.settings_service = ProjectSettingsService(self.project)
self._terminate = False
def __init__(self,
project,
tracking_id: str = None,
request_timeout: float = None):
self.project = project
self.settings_service = ProjectSettingsService(self.project)
self.tracking_id = tracking_id or self.settings_service.get(
"tracking_ids.cli")
self.request_timeout = request_timeout or REQUEST_TIMEOUT
self.send_anonymous_usage_stats = self.settings_service.get(
"send_anonymous_usage_stats")
self.project_id = self.load_project_id()
self.client_id = self.load_client_id()
def cli(ctx, log_level, verbose):
"""
Get help at https://www.meltano.com/docs/command-line-interface.html
"""
if log_level:
ProjectSettingsService.config_override["cli.log_level"] = log_level
ctx.ensure_object(dict)
ctx.obj["verbosity"] = verbose
try:
project = Project.find()
setup_logging(project)
readonly = ProjectSettingsService(project).get("project_readonly")
if readonly:
project.readonly = True
if project.readonly:
logger.debug("Project is read-only.")
ctx.obj["project"] = project
except ProjectNotFound as err:
ctx.obj["project"] = None
except IncompatibleVersionError as err:
click.secho(
"This Meltano project is incompatible with this version of `meltano`.",
fg="yellow",
)
click.echo(
"For more details, visit http://meltano.com/docs/installation.html#upgrading-meltano-version"
)
sys.exit(3)
class ProjectSettings:
settings_map = {
"SERVER_NAME": "ui.server_name",
"SESSION_COOKIE_DOMAIN": "ui.session_cookie_domain",
"SESSION_COOKIE_SECURE": "ui.session_cookie_secure",
"SECRET_KEY": "ui.secret_key",
# Flask-Security
"SECURITY_PASSWORD_SALT": "ui.password_salt",
# Flask-SQLAlchemy
"SQLALCHEMY_DATABASE_URI": "database_uri",
# Flask-Authlib
"GITLAB_CLIENT_ID": "oauth.gitlab.client_id",
"GITLAB_CLIENT_SECRET": "oauth.gitlab.client_secret",
# Flask-Mail
"MAIL_SERVER": "mail.server",
"MAIL_PORT": "mail.port",
"MAIL_DEFAULT_SENDER": "mail.default_sender",
"MAIL_USE_TLS": "mail.use_tls",
"MAIL_USERNAME": "******",
"MAIL_PASSWORD": "******",
"MAIL_DEBUG": "mail.debug",
}
def __init__(self, project: Project):
self.settings_service = ProjectSettingsService(project)
def as_dict(self):
return {
config_key: self.settings_service.get(setting_name)
for config_key, setting_name in self.settings_map.items()
}
def test_activate_project_readonly_dotenv(self, project, cli_runner, pushd,
monkeypatch):
ProjectSettingsService(project).set("project_readonly", True)
assert Project._default is None
pushd(project.root)
cli_runner.invoke(cli, ["discover"])
assert Project._default.readonly == True
def identity():
project = Project.find()
settings_service = ProjectSettingsService(project)
if current_user.is_anonymous:
return jsonify({
"username":
"******",
"anonymous":
True,
"can_sign_in":
settings_service.get("ui.authentication"),
})
return jsonify({
"username": current_user.username,
"anonymous": False,
"can_sign_in": False
})
def setup_security(app, project):
options = {
"login_form": MeltanoLoginForm,
"register_form": MeltanoRegisterFrom,
"confirm_register_form": MeltanoConfirmRegisterForm,
}
settings_service = ProjectSettingsService(project)
if not settings_service.get("ui.authentication"):
# the FreeUser is free to do everything and has all
# roles and permissions automatically.
options["anonymous_user"] = FreeUser
else:
# Use Flask's built-in AnonymousUser which is not deemed to be authenticated
# and has no roles
pass
security.init_app(app, users, **options)
security.unauthorized_handler(unauthorized_callback)
user_logged_in.connect_via(app)(_user_logged_in_hook)
identity_loaded.connect_via(app)(_identity_loaded_hook)
def ensure_secure_setup(project):
settings_service = ProjectSettingsService(project)
if not settings_service.get("ui.authentication"):
return
facts = []
if (
settings_service.get("ui.server_name") is None
and settings_service.get("ui.session_cookie_domain") is None
):
facts.append(
f"- Neither the 'ui.server_name' or 'ui.session_cookie_domain' setting has been set"
)
secure_settings = ["ui.secret_key", "ui.password_salt"]
for setting_name in secure_settings:
value, source = settings_service.get_with_source(setting_name)
if source is SettingValueStore.DEFAULT:
facts.append(
f"- The '{setting_name}' setting has not been changed from the default test value"
)
if facts:
click.secho(
"Authentication is enabled, but your configuration is currently insecure:",
fg="red",
)
for fact in facts:
click.echo(fact)
click.echo(
"For more information about these settings and how to set them, visit https://www.meltano.com/docs/settings.html#ui-authentication"
)
click.echo()
def passes_authentication_checks():
project = Project.find()
settings_service = ProjectSettingsService(project)
if not settings_service.get("ui.authentication"):
logging.debug(f"Authentication not required because it's disabled")
return True
if current_user.is_authenticated:
logging.debug(f"Authenticated as '{current_user.username}'")
return True
if settings_service.get(
"ui.anonymous_readonly") and current_user.is_anonymous:
# The `@roles_required("admin")` and `@block_if_readonly` checks
# will take care of enforcing authentication as appropriate
logging.debug(
f"Authentication not required because anonymous users have read-only access"
)
return True
return False
def config(ctx, project, plugin_type, plugin_name, format, extras):
plugin_type = PluginType.from_cli_argument(
plugin_type) if plugin_type else None
plugins_service = ProjectPluginsService(project)
try:
plugin = plugins_service.find_plugin(plugin_name,
plugin_type=plugin_type,
configurable=True)
except PluginNotFoundError:
if plugin_name == "meltano":
plugin = None
else:
raise
_, Session = project_engine(project)
session = Session()
try:
if plugin:
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service)
else:
settings = ProjectSettingsService(
project, config_service=plugins_service.config_service)
ctx.obj["settings"] = settings
ctx.obj["session"] = session
if ctx.invoked_subcommand is None:
if format == "json":
process = extras is not True
config = settings.as_dict(extras=extras,
process=process,
session=session)
print(json.dumps(config, indent=2))
elif format == "env":
env = settings.as_env(extras=extras, session=session)
with tempfile.NamedTemporaryFile() as temp_dotenv:
path = temp_dotenv.name
for key, value in env.items():
dotenv.set_key(path, key, value)
dotenv_content = Path(temp_dotenv.name).read_text()
print(dotenv_content, end="")
finally:
session.close()
def setup(ctx, server_name, **flags):
"""
Generates and stores server name and secrets.
"""
project = ctx.obj["project"]
settings_service = ProjectSettingsService(project)
def set_setting_env(setting_name, value):
settings_service.set(setting_name, value, store=SettingValueStore.DOTENV)
set_setting_env("ui.server_name", server_name)
ui_cfg_path = project.root_dir("ui.cfg")
if ui_cfg_path.exists():
raise CliError(
f"Found existing secrets in file '{ui_cfg_path}'. Please delete this file and rerun this command to regenerate the secrets."
)
generate_secret = lambda: secrets.token_hex(int(flags["bits"] / 8)) # in bytes
secret_settings = ["ui.secret_key", "ui.password_salt"]
for setting_name in secret_settings:
value, source = settings_service.get_with_source(setting_name)
if source is not SettingValueStore.DEFAULT:
click.echo(
f"Setting '{setting_name}' has already been set in {source.label}. Please unset it manually and rerun this command to regenerate this secret."
)
else:
set_setting_env(setting_name, generate_secret())
click.echo(
"The server name and generated secrets have been stored in your project's `.env` file."
)
click.echo(
"In production, you will likely want to move these settings to actual environment variables, since `.env` is in `.gitignore` by default."
)
class SingerRunner(Runner):
def __init__(self, elt_context: ELTContext):
self.context = elt_context
self.project_settings_service = ProjectSettingsService(
self.context.project,
config_service=self.context.plugins_service.config_service,
)
def stop(self, process, **wait_args):
while True:
try:
code = process.wait(**wait_args)
logging.debug(f"{process} exited with {code}")
return code
except subprocess.TimeoutExpired:
process.kill()
logging.error(f"{process} was killed.")
async def invoke( # noqa: WPS217, WPS210, WPS211, WPS213, WPS231
self,
tap: PluginInvoker,
target: PluginInvoker,
extractor_log=None,
loader_log=None,
extractor_out=None,
loader_out=None,
):
"""Invoke tap and target together."""
extractor_log = extractor_log or sys.stderr
loader_log = loader_log or sys.stderr
# The StreamReader line length limit also acts as half the buffer size,
# which cannot be set directly:
# - https://github.com/python/cpython/blob/v3.8.7/Lib/asyncio/streams.py#L395-L396
# - https://github.com/python/cpython/blob/v3.8.7/Lib/asyncio/streams.py#L482
stream_buffer_size = self.project_settings_service.get("elt.buffer_size")
line_length_limit = stream_buffer_size // 2
# Start tap
try:
p_tap = await tap.invoke_async(
limit=line_length_limit,
stdout=asyncio.subprocess.PIPE, # Singer messages
stderr=asyncio.subprocess.PIPE, # Log
)
except Exception as err:
raise RunnerError(f"Cannot start extractor: {err}") from err
# Start target
try:
p_target = await target.invoke_async(
limit=line_length_limit,
stdin=asyncio.subprocess.PIPE, # Singer messages
stdout=asyncio.subprocess.PIPE, # Singer state
stderr=asyncio.subprocess.PIPE, # Log
)
except Exception as err:
raise RunnerError(f"Cannot start loader: {err}") from err
# Process tap output
tap_outputs = [p_target.stdin]
if extractor_out:
tap_outputs.insert(0, extractor_out)
tap_stdout_future = asyncio.ensure_future(
capture_subprocess_output(p_tap.stdout, *tap_outputs)
)
tap_stderr_future = asyncio.ensure_future(
capture_subprocess_output(p_tap.stderr, extractor_log)
)
# Process target output
target_outputs = [self.bookmark_writer()]
if loader_out:
target_outputs.insert(0, loader_out)
target_stdout_future = asyncio.ensure_future(
capture_subprocess_output(p_target.stdout, *target_outputs)
)
target_stderr_future = asyncio.ensure_future(
capture_subprocess_output(p_target.stderr, loader_log)
)
# Wait for tap or target to complete, or for one of the output handlers to raise an exception.
tap_process_future = asyncio.ensure_future(p_tap.wait())
target_process_future = asyncio.ensure_future(p_target.wait())
output_exception_future = asyncio.ensure_future(
asyncio.wait(
[
tap_stdout_future,
tap_stderr_future,
target_stdout_future,
target_stderr_future,
],
return_when=asyncio.FIRST_EXCEPTION,
),
)
done, _ = await asyncio.wait(
[tap_process_future, target_process_future, output_exception_future],
return_when=asyncio.FIRST_COMPLETED,
)
# If `output_exception_future` completes first, one of the output handlers raised an exception or all completed successfully.
if output_exception_future in done:
output_futures_done, _ = output_exception_future.result()
output_futures_failed = [
future
for future in output_futures_done
if future.exception() is not None
]
if output_futures_failed:
# If any output handler raised an exception, re-raise it.
# Special behavior for the tap stdout handler raising a line length limit error.
if tap_stdout_future in output_futures_failed:
self._handle_tap_line_length_limit_error(
tap_stdout_future.exception(),
line_length_limit=line_length_limit,
stream_buffer_size=stream_buffer_size,
)
failed_future = output_futures_failed.pop()
raise failed_future.exception()
else:
# If all of the output handlers completed without raising an exception,
# we still need to wait for the tap or target to complete.
done, _ = await asyncio.wait(
[tap_process_future, target_process_future],
return_when=asyncio.FIRST_COMPLETED,
)
if target_process_future in done:
target_code = target_process_future.result()
if tap_process_future in done:
tap_code = tap_process_future.result()
else:
# If the target completes before the tap, it failed before processing all tap output
# Kill tap and cancel output processing since there's no more target to forward messages to
p_tap.kill()
tap_stdout_future.cancel()
tap_stderr_future.cancel()
# Pretend the tap finished successfully since it didn't itself fail
tap_code = 0
# Wait for all buffered target output to be processed
await asyncio.wait([target_stdout_future, target_stderr_future])
else: # if tap_process_future in done:
# If the tap completes before the target, the target should have a chance to process all tap output
tap_code = tap_process_future.result()
# Wait for all buffered tap output to be processed
await asyncio.wait([tap_stdout_future, tap_stderr_future])
# Close target stdin so process can complete naturally
p_target.stdin.close()
# Wait for all buffered target output to be processed
await asyncio.wait([target_stdout_future, target_stderr_future])
# Wait for target to complete
target_code = await target_process_future
if tap_code and target_code:
raise RunnerError(
"Extractor and loader failed",
{PluginType.EXTRACTORS: tap_code, PluginType.LOADERS: target_code},
)
elif tap_code:
raise RunnerError("Extractor failed", {PluginType.EXTRACTORS: tap_code})
elif target_code:
raise RunnerError("Loader failed", {PluginType.LOADERS: target_code})
def bookmark_writer(self):
incomplete_state = self.context.full_refresh and self.context.select_filter
payload_flag = Payload.INCOMPLETE_STATE if incomplete_state else Payload.STATE
return BookmarkWriter(
self.context.job, self.context.session, payload_flag=payload_flag
)
def dry_run(self, tap: PluginInvoker, target: PluginInvoker):
logging.info("Dry run:")
logging.info(f"\textractor: {tap.plugin.name} at '{tap.exec_path()}'")
logging.info(f"\tloader: {target.plugin.name} at '{target.exec_path()}'")
async def run(
self, extractor_log=None, loader_log=None, extractor_out=None, loader_out=None
):
tap = self.context.extractor_invoker()
target = self.context.loader_invoker()
if self.context.dry_run:
return self.dry_run(tap, target)
with tap.prepared(self.context.session), target.prepared(self.context.session):
await self.invoke(
tap,
target,
extractor_log=extractor_log,
loader_log=loader_log,
extractor_out=extractor_out,
loader_out=loader_out,
)
def _handle_tap_line_length_limit_error(
self,
exception,
line_length_limit,
stream_buffer_size,
):
# StreamReader.readline can raise a ValueError wrapping a LimitOverrunError:
# https://github.com/python/cpython/blob/v3.8.7/Lib/asyncio/streams.py#L549
if not isinstance(exception, ValueError):
return
exception = exception.__context__ # noqa: WPS609
if not isinstance(exception, asyncio.LimitOverrunError):
return
logging.error(
f"The extractor generated a message exceeding the message size limit of {human_size(line_length_limit)} (half the buffer size of {human_size(stream_buffer_size)})."
)
logging.error(
"To let this message be processed, increase the 'elt.buffer_size' setting to at least double the size of the largest expected message, and try again."
)
logging.error(
"To learn more, visit https://www.meltano.com/docs/settings.html#elt-buffer-size"
)
raise RunnerError("Output line length limit exceeded") from exception
class GoogleAnalyticsTracker:
def __init__(self,
project,
tracking_id: str = None,
request_timeout: float = None):
self.project = project
self.settings_service = ProjectSettingsService(self.project)
self.tracking_id = tracking_id or self.settings_service.get(
"tracking_ids.cli")
self.request_timeout = request_timeout or REQUEST_TIMEOUT
self.send_anonymous_usage_stats = self.settings_service.get(
"send_anonymous_usage_stats")
self.project_id = self.load_project_id()
self.client_id = self.load_client_id()
def load_project_id(self) -> uuid.UUID:
"""
Fetch the project_id from the project config file.
If it is not found (e.g. first time run), generate a valid uuid4 and
store it in the project config file.
"""
try:
project_id_str = self.settings_service.get("project_id")
project_id = uuid.UUID(project_id_str or "", version=4)
except ValueError:
project_id = uuid.uuid4()
if self.send_anonymous_usage_stats:
# If we are set to track Anonymous Usage stats, also store
# the generated project_id back to the project config file
# so that it persists between meltano runs.
self.settings_service.set("project_id", str(project_id))
return project_id
def load_client_id(self) -> uuid.UUID:
"""
Fetch the client_id from the non-versioned analytics.json.
If it is not found (e.g. first time run), generate a valid uuid4 and
store it in analytics.json.
"""
file_path = self.project.meltano_dir().joinpath("analytics.json")
try:
with open(file_path) as file:
file_data = json.load(file)
client_id_str = file_data["client_id"]
client_id = uuid.UUID(client_id_str, version=4)
except FileNotFoundError:
client_id = uuid.uuid4()
if self.send_anonymous_usage_stats:
# If we are set to track Anonymous Usage stats, also store
# the generated client_id in a non-versioned analytics.json file
# so that it persists between meltano runs.
with open(file_path, "w") as file:
data = {"client_id": str(client_id)}
json.dump(data, file)
return client_id
def event(self, category: str, action: str) -> Dict:
"""Constract a GA event with all the required parameters."""
event = {
"v": "1",
"tid": self.tracking_id,
"cid": self.client_id,
"ds": "meltano cli",
"t": "event",
"ec": category,
"ea": action,
"el": self.project_id,
"cd1": self.project_id, # maps to the custom dimension 1 of the UI
}
return event
def track_data(self, data: Dict, debug: bool = False) -> None:
"""Send usage statistics back to Google Analytics."""
if self.send_anonymous_usage_stats == False:
# Only send anonymous usage stats if you have explicit permission
return
if debug:
tracking_uri = DEBUG_MEASUREMENT_PROTOCOL_URI
else:
tracking_uri = MEASUREMENT_PROTOCOL_URI
try:
r = requests.post(tracking_uri,
data=data,
timeout=self.request_timeout)
if debug:
logging.debug("GoogleAnalyticsTracker.track_data:")
logging.debug(data)
logging.debug("Response:")
logging.debug(f"status_code: {r.status_code}")
logging.debug(r.text)
except requests.exceptions.Timeout:
logging.debug(
"GoogleAnalyticsTracker.track_data: Request Timed Out")
except requests.exceptions.ConnectionError as e:
logging.debug("GoogleAnalyticsTracker.track_data: ConnectionError")
logging.debug(e)
except requests.exceptions.RequestException as e:
logging.debug(
"GoogleAnalyticsTracker.track_data: RequestException")
logging.debug(e)
def track_event(self,
category: str,
action: str,
debug: bool = False) -> None:
event = self.event(category, action)
self.track_data(event, debug)
def track_meltano_init(self,
project_name: str,
debug: bool = False) -> None:
event = self.track_event(category="meltano init",
action=f"meltano init {project_name}",
debug=debug)
def track_meltano_add(self,
plugin_type: str,
plugin_name: str,
debug: bool = False) -> None:
event = self.track_event(
category=f"meltano add {plugin_type}",
action=f"meltano add {plugin_type} {plugin_name}",
debug=debug,
)
def track_meltano_discover(self,
plugin_type: str,
debug: bool = False) -> None:
event = self.track_event(
category="meltano discover",
action=f"meltano discover {plugin_type}",
debug=debug,
)
def track_meltano_elt(self,
extractor: str,
loader: str,
transform: str,
debug: bool = False) -> None:
event = self.track_event(
category="meltano elt",
action=f"meltano elt {extractor} {loader} --transform {transform}",
debug=debug,
)
def track_meltano_install(self, debug: bool = False) -> None:
event = self.track_event(category="meltano install",
action="meltano install",
debug=debug)
def track_meltano_invoke(self,
plugin_name: str,
plugin_args: str,
debug: bool = False) -> None:
event = self.track_event(
category="meltano invoke",
action=f"meltano invoke {plugin_name} {plugin_args}",
debug=debug,
)
def track_meltano_schedule(self, schedule, debug=False) -> None:
self.track_event(
category="meltano schedule",
action=
f"meltano schedule {schedule.name} {schedule.extractor} {schedule.loader} {schedule.interval} --transform={schedule.transform}",
debug=debug,
)
def track_meltano_select(
self,
extractor: str,
entities_filter: str,
attributes_filter: str,
flags: [],
debug: bool = False,
) -> None:
action = f"meltano select {extractor} {entities_filter} {attributes_filter}"
if flags["list"]:
action = action + " --list"
if flags["all"]:
action = action + " --all"
if flags["exclude"]:
action = action + " --exclude"
event = self.track_event(category="meltano select",
action=action,
debug=debug)
def track_meltano_ui(self, debug: bool = False) -> None:
action = f"meltano ui"
event = self.track_event(category="meltano ui",
action=action,
debug=debug)
def subject(project):
return ProjectSettingsService(project)
def create_app(config={}):
project = Project.find()
setup_logging(project)
settings_service = ProjectSettingsService(project)
project_engine(project, default=True)
app = Flask(
__name__, instance_path=str(project.root), instance_relative_config=True
)
# make sure we have the latest environment loaded
importlib.reload(meltano.api.config)
app.config.from_object("meltano.api.config")
app.config.from_mapping(**meltano.api.config.ProjectSettings(project).as_dict())
app.config.from_mapping(**config)
# File logging
file_handler = logging.handlers.RotatingFileHandler(
str(project.run_dir("meltano-ui.log")), backupCount=3
)
formatter = logging.Formatter(fmt=FORMAT)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# 1) Extensions
security_options = {}
from .models import db
from .mail import mail
from .executor import setup_executor
from .security import security, users, setup_security
from .security.oauth import setup_oauth
from .json import setup_json
db.init_app(app)
mail.init_app(app)
setup_executor(app, project)
setup_security(app, project)
setup_oauth(app)
setup_json(app)
# we need to setup CORS for development
if app.env == "development":
CORS(app, origins="http://localhost:8080", supports_credentials=True)
# 2) Register the URL Converters
from .url_converters import PluginRefConverter
app.url_map.converters["plugin_ref"] = PluginRefConverter
# 3) Register the controllers
from .controllers.dashboards import dashboardsBP
from .controllers.embeds import embedsBP
from .controllers.reports import reportsBP
from .controllers.repos import reposBP
from .controllers.settings import settingsBP
from .controllers.sql import sqlBP
from .controllers.orchestrations import orchestrationsBP
from .controllers.plugins import pluginsBP
from .controllers.root import root, api_root
app.register_blueprint(dashboardsBP)
app.register_blueprint(embedsBP)
app.register_blueprint(reportsBP)
app.register_blueprint(reposBP)
app.register_blueprint(settingsBP)
app.register_blueprint(sqlBP)
app.register_blueprint(orchestrationsBP)
app.register_blueprint(pluginsBP)
app.register_blueprint(root)
app.register_blueprint(api_root)
if app.config["PROFILE"]:
from .profiler import init
init(app)
# Notifications
if settings_service.get("ui.notification"):
from .events import notifications
notifications.init_app(app)
logger.debug("Notifications are enabled.")
else:
logger.debug("Notifications are disabled.")
# Google Analytics setup
tracker = GoogleAnalyticsTracker(project)
@app.before_request
def setup_js_context():
# setup the appUrl
appUrl = urlsplit(request.host_url)
g.jsContext = {"appUrl": appUrl.geturl()[:-1], "version": meltano.__version__}
setting_map = {
"isSendAnonymousUsageStats": "send_anonymous_usage_stats",
"projectId": "project_id",
"trackingID": "tracking_ids.ui",
"embedTrackingID": "tracking_ids.ui_embed",
"isProjectReadonlyEnabled": "project_readonly",
"isReadonlyEnabled": "ui.readonly",
"isAnonymousReadonlyEnabled": "ui.anonymous_readonly",
"isNotificationEnabled": "ui.notification",
"isAnalysisEnabled": "ui.analysis",
"logoUrl": "ui.logo_url",
"oauthServiceUrl": "oauth_service.url",
}
for context_key, setting_name in setting_map.items():
g.jsContext[context_key] = settings_service.get(setting_name)
providers = settings_service.get("oauth_service.providers")
g.jsContext["oauthServiceProviders"] = [
provider for provider in providers.split(",") if provider
]
@app.after_request
def after_request(res):
res.headers[VERSION_HEADER] = meltano.__version__
return res
@app.errorhandler(500)
def internal_error(exception):
logger.info(f"Error: {exception}")
return jsonify({"error": True, "code": str(exception)}), 500
@app.errorhandler(ProjectReadonly)
def _handle(ex):
return (jsonify({"error": True, "code": str(ex)}), HTTP_READONLY_CODE)
# create the dispatcher to host the `OAuthService`
app.wsgi_app = DispatcherMiddleware(
app.wsgi_app, {"/-/oauth": create_oauth_service()}
)
return app
def __init__(self, project: Project):
self.settings_service = ProjectSettingsService(project)
def test_tracking_disabled(project):
assert os.getenv("MELTANO_DISABLE_TRACKING") == "True"
assert ProjectSettingsService(project).get(
"send_anonymous_usage_stats") == False
def __init__(self, project: Project):
self.project = project
self.project_id = ProjectSettingsService(
self.project).get("project_id")
import importlib
import logging
import os
import warnings
import meltano
from gunicorn.glogging import CONFIG_DEFAULTS
from meltano.core.logging.utils import FORMAT
from meltano.core.project import Project
from meltano.core.project_settings_service import ProjectSettingsService
_project = Project.find()
_settings_service = ProjectSettingsService(_project)
loglevel = _settings_service.get("cli.log_level")
logconfig_dict = CONFIG_DEFAULTS.copy()
logconfig_dict["loggers"]["gunicorn.access"]["propagate"] = False
logconfig_dict["loggers"]["gunicorn.error"]["propagate"] = False
logconfig_dict["formatters"]["generic"] = {"format": FORMAT}
if loglevel != "debug":
logconfig_dict["loggers"].pop("gunicorn.access")
_bind_host = _settings_service.get("ui.bind_host")
_bind_port = _settings_service.get("ui.bind_port")
bind = f"{_bind_host}:{_bind_port}"
workers = _settings_service.get("ui.workers")
forwarded_allow_ips = _settings_service.get("ui.forwarded_allow_ips")