def config_metadata_rules(config):
rules = []
for key, value in config.items():
if not key.startswith("metadata."):
continue
# metadata.<tap_stream_id>.<key>
# metadata.<tap_stream_id>.<prop>.<key>
# metadata.<tap_stream_id>.<prop>.<subprop>.<key>
# metadata.<tap_stream_id>.properties.<prop>.<key>
# metadata.<tap_stream_id>.properties.<prop>.properties.<subprop>.<key>
_, tap_stream_id, *props, key = key.split(".")
if key == "selected":
value = truthy(value)
rules.append(
MetadataRule(
tap_stream_id=tap_stream_id,
breadcrumb=property_breadcrumb(props),
key=key,
value=value,
)
)
return rules
def version():
response_payload = {"version": meltano.__version__}
if truthy(request.args.get("include_latest")):
res = requests.get("https://pypi.org/pypi/meltano/json")
pypi_payload = res.json()
response_payload["latest_version"] = pypi_payload["info"]["version"]
return jsonify(response_payload)
def __init__(self, project, tracking_id: str = None, request_timeout: float = None):
self.project = project
self.tracking_id = tracking_id or MELTANO_TRACKING_ID
self.request_timeout = request_timeout or REQUEST_TIMEOUT
self.send_anonymous_usage_stats = (
not truthy(os.getenv("MELTANO_DISABLE_TRACKING"))
and self.project.meltano.get("send_anonymous_usage_stats", False) == True
)
self.project_id = self.load_project_id()
self.client_id = self.load_client_id()
def ui(project, reload, bind_port, bind):
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_ui()
loop = asyncio.get_event_loop()
# we need to prime the ChildWatcher here so we can
# call subprocesses asynchronously from threads
#
# see https://docs.python.org/3/library/asyncio-subprocess.html#subprocess-and-threads
# TODO: remove when running on Python 3.8
asyncio.get_child_watcher()
workers = []
if not truthy(os.getenv("MELTANO_DISABLE_AIRFLOW", False)):
workers.append(AirflowWorker(project))
workers.append(MeltanoCompilerWorker(project))
# we need to whitelist the loaders here because not
# all the loaders support dbt in the first place
dbt_docs_loader = os.getenv("MELTANO_DBT_DOCS_LOADER", "target-postgres")
if dbt_docs_loader:
workers.append(DbtWorker(project, dbt_docs_loader, loop=loop))
else:
logging.info(
"No loader enabled for dbt docs generation, set the MELTANO_DBT_DOCS_LOADER variable to enable one."
)
workers.append(UIAvailableWorker("http://localhost:{bind_port}"))
workers.append(
APIWorker(
project,
f"{bind}:{bind_port}",
reload=reload or os.getenv("FLASK_ENV") == "development",
))
cleanup = start_workers(workers)
def handle_terminate(signal, frame):
cleanup()
signal.signal(signal.SIGTERM, handle_terminate)
logger.info("All workers started.")
def start(ctx, reload, bind_port, bind):
project = ctx.obj["project"]
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_ui()
workers = []
if not truthy(os.getenv("MELTANO_DISABLE_AIRFLOW", False)):
try:
config_service = ConfigService(project)
config_service.find_plugin("airflow")
workers.append(AirflowWorker(project))
except PluginMissingError:
pass
try:
compiler_worker = MeltanoCompilerWorker(project)
compiler_worker.compiler.compile()
workers.append(compiler_worker)
except Exception as e:
logger.error(f"Initial compilation failed: {e}")
workers.append(UIAvailableWorker("http://localhost:{bind_port}"))
workers.append(
APIWorker(
project,
f"{bind}:{bind_port}",
reload=reload or os.getenv("FLASK_ENV") == "development",
))
cleanup = start_workers(workers)
def handle_terminate(signal, frame):
cleanup()
signal.signal(signal.SIGTERM, handle_terminate)
logger.info("All workers started.")
import os
import logging
import datetime
from meltano.core.utils import truthy
from meltano.api.headers import *
from meltano.core.tracking.ga_tracker import (
MELTANO_UI_TRACKING_ID,
MELTANO_EMBED_TRACKING_ID,
)
# Flask
# -----------------
THREADS_PER_PAGE = 1
PROFILE = truthy(os.getenv("FLASK_PROFILE"))
## Change this value in production
SECRET_KEY = "thisisnotapropersecretkey"
# Meltano
# -----------------
MELTANO_AUTHENTICATION = truthy(os.getenv("MELTANO_AUTHENTICATION"))
MELTANO_NOTIFICATION = truthy(os.getenv("MELTANO_NOTIFICATION"))
MELTANO_OAUTH_SERVICE_URL = os.getenv("MELTANO_OAUTH_SERVICE_URL", None)
MELTANO_OAUTH_SERVICE_PROVIDERS = [
provider for provider in (
os.getenv("MELTANO_OAUTH_SERVICE_PROVIDERS", "").split(","))
if provider
]
MELTANO_READONLY = truthy(os.getenv("MELTANO_READONLY"))
MELTANO_UI_URL = os.getenv("MELTANO_UI_URL", "/")
import datetime
import logging
import os
from meltano.api.headers import *
from meltano.core.project import Project
from meltano.core.project_settings_service import (
ProjectSettingsService,
SettingValueStore,
)
from meltano.core.utils import truthy
# Flask
# -----------------
THREADS_PER_PAGE = 1
PROFILE = truthy(os.getenv("FLASK_PROFILE"))
# Meltano
# -----------------
MELTANO_UI_URL = os.getenv("MELTANO_UI_URL", "/")
API_ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
TEMP_FOLDER = os.path.join(API_ROOT_DIR, "static/tmp")
PROJECT_ROOT_DIR = os.path.dirname(API_ROOT_DIR)
# Flask-SQLAlchemy
# -----------------
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
# Flask-Security
def cast_value(self, setting_def, value):
if isinstance(value, str) and setting_def.kind == "boolean":
value = truthy(value)
return value
import os
import datetime
from meltano.core.utils import truthy
# Flask
# -----------------
THREADS_PER_PAGE = 1
PROFILE = truthy(os.getenv("FLASK_PROFILE"))
## Change this value in production
SECRET_KEY = "483be43cf29204e24d85cf711e36ea978a4d0ab316d8ecd7ae1ce5ecff3e29c1"
# Meltano
# -----------------
MELTANO_AUTHENTICATION = truthy(os.getenv("MELTANO_AUTHENTICATION"))
MELTANO_UI_URL = os.getenv("MELTANO_UI_URL", "")
AIRFLOW_DISABLED = truthy(os.getenv("MELTANO_DISABLE_AIRFLOW"))
API_ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
TEMP_FOLDER = os.path.join(API_ROOT_DIR, "static/tmp")
PROJECT_ROOT_DIR = os.path.dirname(API_ROOT_DIR)
# Flask-SQLAlchemy
# -----------------
SQLALCHEMY_ECHO = False
SQLALCHEMY_TRACK_MODIFICATIONS = False
SQLALCHEMY_DATABASE_URI = os.getenv("MELTANO_DATABASE_URI")
# Flask-security
# -----------------