def validate_plugin_config(plugin: PluginRef, name, value, project: Project,
settings: PluginSettingsService):
setting_def = settings.find_setting(plugin, name)
# we want to prevent the edition of protected settings from the UI
if setting_def.protected:
logging.warning("Cannot set a 'protected' configuration externally.")
return False
if setting_def.kind == "file" and value and value != "":
uploads_directory = project.extract_dir(plugin.full_name)
resolved_file_path = project.root_dir(value).resolve()
if not str(resolved_file_path).startswith(
str(uploads_directory) + "/"):
logging.warning(
"Cannot set a file configuration to a path outside the project directory"
)
return False
old_value, source = settings.get_value(db.session, plugin, name)
if source in (PluginSettingValueSource.ENV,
PluginSettingValueSource.MELTANO_YML):
logging.warning(
"Cannot override a configuration set in the environment or meltano.yml."
)
return False
return True
def project(self, test_dir, project_init_service):
"""This fixture returns the non-activated project."""
project = project_init_service.init(activate=False, add_discovery=True)
yield project
Project.deactivate()
shutil.rmtree(project.root)
def test_activate(self, project):
Project.deactivate()
assert Project._default is None
Project.activate(project)
assert Project._default is project
assert Project.find() is project
def project(self, project):
Project.deactivate()
monkeypatch = MonkeyPatch()
monkeypatch.setenv(PROJECT_READONLY_ENV, "true")
yield project
monkeypatch.undo()
def test_activate(self, project):
assert os.getenv("MELTANO_PROJECT") is None
with open(".env", "w") as env:
env.write(f"MELTANO_PROJECT={project.root}")
# `Project.find()` always return the default instance
Project.activate(project)
assert os.getenv("MELTANO_PROJECT") == str(project.root)
assert Project.find() is project
def job_state() -> Response:
"""
Endpoint for getting the status of N jobs
"""
project = Project.find()
poll_payload = request.get_json()
job_ids = poll_payload["job_ids"]
jobs = []
for job_id in job_ids:
finder = JobFinder(job_id)
state_job = finder.latest(db.session)
# Validate existence first as a job may not be queued yet as a result of
# another prerequisite async process (dbt installation for example)
if state_job:
state_job_success = finder.latest_success(db.session)
jobs.append({
"job_id":
job_id,
"is_complete":
state_job.is_complete(),
"has_error":
state_job.has_error(),
"started_at":
state_job.started_at,
"ended_at":
state_job.ended_at,
"has_ever_succeeded":
state_job_success.is_success() if state_job_success else None,
})
return jsonify({"jobs": jobs})
def install_batch():
payload = request.get_json()
plugin_type = PluginType(payload["plugin_type"])
plugin_name = payload["name"]
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.find_plugin(plugin_name, plugin_type=plugin_type)
add_service = ProjectAddService(project, plugins_service=plugins_service)
related_plugins = add_service.add_related(plugin)
# We will install the plugins in reverse order, since dependencies
# are listed after their dependents in `related_plugins`, but should
# be installed first.
related_plugins.reverse()
install_service = PluginInstallService(project,
plugins_service=plugins_service)
install_status = install_service.install_plugins(
related_plugins, reason=PluginInstallReason.ADD)
for error in install_status["errors"]:
raise PluginInstallError(error["message"])
return jsonify([plugin.canonical() for plugin in related_plugins])
def cli(ctx, log_level, verbose):
"""
Get help at https://www.meltano.com/docs/command-line-interface.html
"""
if log_level:
ProjectSettingsService.config_override["cli.log_level"] = log_level
ctx.ensure_object(dict)
ctx.obj["verbosity"] = verbose
try:
project = Project.find()
setup_logging(project)
readonly = ProjectSettingsService(project).get("project_readonly")
if readonly:
project.readonly = True
if project.readonly:
logger.debug("Project is read-only.")
ctx.obj["project"] = project
except ProjectNotFound as err:
ctx.obj["project"] = None
except IncompatibleVersionError as err:
click.secho(
"This Meltano project is incompatible with this version of `meltano`.",
fg="yellow",
)
click.echo(
"For more details, visit http://meltano.com/docs/installation.html#upgrading-meltano-version"
)
sys.exit(3)
def save_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
plugin = ConfigService(project).get_plugin(plugin_ref)
# TODO iterate pipelines and save each, also set this connector's profile (reuse `pipelineInFocusIndex`?)
settings = PluginSettingsService(project, show_hidden=False)
for profile in payload:
# select the correct profile
name = profile["name"]
plugin.use_profile(plugin.get_profile(name))
for name, value in profile["config"].items():
if not validate_plugin_config(plugin, name, value, project,
settings):
continue
if value == "":
settings.unset(db.session, plugin, name)
else:
settings.set(db.session, plugin, name, value)
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify(profiles)
def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration profiles
"""
project = Project.find()
settings = PluginSettingsService(project, show_hidden=False)
plugin = ConfigService(project).get_plugin(plugin_ref)
discovery_service = PluginDiscoveryService(project)
try:
plugin_def = discovery_service.find_plugin(plugin.type, plugin.name)
settings_group_validation = plugin_def.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
profiles = settings.profiles_with_config(db.session, plugin, redacted=True)
for profile in profiles:
freeze_profile_config_keys(profile)
return jsonify({
"profiles":
profiles,
"settings":
Canonical.as_canonical(settings.definitions(plugin)),
"settings_group_validation":
settings_group_validation,
})
def decorated(*args, **kwargs):
project = Project.find()
settings_service = ProjectSettingsService(project)
if settings_service.get("ui.readonly"):
return (
jsonify({
"error": True,
"code": "Meltano UI is running in read-only mode"
}),
HTTP_READONLY_CODE,
)
if settings_service.get(
"ui.anonymous_readonly") and current_user.is_anonymous:
return (
jsonify({
"error":
True,
"code":
"Meltano UI is running in read-only mode until you sign in",
}),
HTTP_READONLY_CODE,
)
return f(*args, **kwargs)
def get_report_resource(self, resource_id, today=None):
project = Project.find()
reports_service = ReportsService(project)
report = reports_service.get_report(resource_id)
reports_helper = ReportsHelper()
return reports_helper.get_report_with_query_results(report, today=today)
def get_sql(namespace, topic_name, design_name):
sqlHelper = SqlHelper()
m5oc = sqlHelper.get_m5oc_topic(namespace, topic_name)
design = m5oc.design(design_name)
incoming_json = request.get_json()
sql_dict = sqlHelper.get_sql(design, incoming_json)
outgoing_sql = sql_dict["sql"]
aggregates = sql_dict["aggregates"]
query_attributes = sql_dict["query_attributes"]
base_dict = {"sql": outgoing_sql, "error": False}
base_dict["query_attributes"] = query_attributes
base_dict["aggregates"] = aggregates
if not incoming_json["run"]:
return jsonify(base_dict)
# we need to find the pipeline that loaded the data for this model
# this is running off the assumption that there is only one pipeline
# that can load data for a specific model
project = Project.find()
schedule_service = ScheduleService(project)
schedule = schedule_service.find_namespace_schedule(
m5oc.content["plugin_namespace"])
results = sqlHelper.get_query_results(schedule.extractor, schedule.loader,
schedule.transform, outgoing_sql)
base_dict["results"] = results
base_dict["empty"] = len(results) == 0
return jsonify(base_dict)
def job_log(job_id) -> Response:
"""
Endpoint for getting the most recent log generated by a job with job_id
"""
project = Project.find()
try:
log_service = JobLoggingService(project)
log = log_service.get_latest_log(job_id)
has_log_exceeded_max_size = False
except SizeThresholdJobLogException as err:
log = None
has_log_exceeded_max_size = True
finder = JobFinder(job_id)
state_job = finder.latest(db.session)
state_job_success = finder.latest_success(db.session)
return jsonify({
"job_id":
job_id,
"log":
log,
"has_log_exceeded_max_size":
has_log_exceeded_max_size,
"has_error":
state_job.has_error() if state_job else False,
"started_at":
state_job.started_at if state_job else None,
"ended_at":
state_job.ended_at if state_job else None,
"trigger":
state_job.trigger if state_job else None,
"has_ever_succeeded":
state_job_success.is_success() if state_job_success else None,
})
def get_dashboards(self):
project = Project.find()
dashboardsParser = M5oCollectionParser(
project.analyze_dir("dashboards"),
M5oCollectionParserTypes.Dashboard)
return dashboardsParser.parse()
def get_pipeline_schedules():
"""
Endpoint for getting the pipeline schedules
"""
project = Project.find()
schedule_service = ScheduleService(project)
schedules = list(map(dict, schedule_service.schedules()))
for schedule in schedules:
finder = JobFinder(schedule["name"])
state_job = finder.latest(db.session)
schedule["has_error"] = state_job.has_error() if state_job else False
schedule["is_running"] = state_job.is_running() if state_job else False
schedule["job_id"] = schedule["name"]
schedule["started_at"] = state_job.started_at if state_job else None
schedule["ended_at"] = state_job.ended_at if state_job else None
schedule["trigger"] = state_job.trigger if state_job else None
state_job_success = finder.latest_success(db.session)
schedule["has_ever_succeeded"] = (state_job_success.is_success()
if state_job_success else None)
schedule["start_date"] = (schedule["start_date"].date().isoformat()
if schedule["start_date"] else None)
return jsonify(schedules)
def installed():
"""Returns JSON of all installed plugins
Fuses the discovery.yml data with meltano.yml data and sorts each type alphabetically by name
"""
project = Project.find()
config = ConfigService(project)
discovery = PluginDiscoveryService(project)
installed_plugins = {}
# merge definitions
for plugin in sorted(config.plugins(), key=lambda x: x.name):
try:
definition = discovery.find_plugin(plugin.type, plugin.name)
merged_plugin_definition = {
**definition.canonical(),
**plugin.canonical()
}
except PluginNotFoundError:
merged_plugin_definition = {**plugin.canonical()}
merged_plugin_definition.pop("settings", None)
merged_plugin_definition.pop("select", None)
if not plugin.type in installed_plugins:
installed_plugins[plugin.type] = []
installed_plugins[plugin.type].append(merged_plugin_definition)
return jsonify({
**project.meltano.canonical(), "plugins": installed_plugins
})
def save_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for persisting a plugin configuration
"""
project = Project.find()
payload = request.get_json()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
config = payload.get("config", {})
for name, value in config.items():
if not validate_plugin_config(plugin, name, value, project, settings):
continue
if value == "":
settings.unset(name, session=db.session)
else:
settings.set(name, value, session=db.session)
return jsonify(get_config_with_metadata(settings))
def get_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for getting a plugin's configuration
"""
project = Project.find()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
try:
settings_group_validation = plugin.settings_group_validation
except PluginNotFoundError:
settings_group_validation = []
return jsonify({
**get_config_with_metadata(settings),
"settings":
Canonical.as_canonical(settings.definitions(extras=False)),
"settings_group_validation":
settings_group_validation,
})
def run():
project = Project.find()
schedule_payload = request.get_json()
name = schedule_payload["name"]
job_id = run_schedule(project, name)
return jsonify({"job_id": job_id}), 202
def index():
project = Project.find()
onlyfiles = [f for f in project.model_dir().iterdir() if f.is_file()]
path = project.model_dir()
dashboardsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Dashboard)
reportsParser = M5oCollectionParser(path, M5oCollectionParserTypes.Report)
reportsFiles = reportsParser.parse()
dashboardFiles = dashboardsParser.parse()
sortedM5oFiles = {
"dashboards": {"label": "Dashboards", "items": dashboardFiles},
"documents": {"label": "Documents", "items": []},
"topics": {"label": "Topics", "items": []},
"reports": {"label": "Reports", "items": reportsFiles},
"tables": {"label": "Tables", "items": []},
}
onlydocs = project.model_dir().parent.glob("*.md")
for d in onlydocs:
file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict(
d.relative_to(project.root), str(d.name)
)
sortedM5oFiles["documents"]["items"].append(file_dict)
for f in onlyfiles:
filename, ext = os.path.splitext(f)
if ext != ".m5o":
continue
# filename splittext twice occurs due to current *.type.extension convention (two dots)
filename = filename.lower()
filename, ext = os.path.splitext(filename)
file_dict = MeltanoAnalysisFileParser.fill_base_m5o_dict(
f.relative_to(project.root), filename
)
if ext == ".topic":
sortedM5oFiles["topics"]["items"].append(file_dict)
if ext == ".table":
sortedM5oFiles["tables"]["items"].append(file_dict)
m5o_parser = MeltanoAnalysisFileParser(project)
for package in m5o_parser.packages():
package_files = MeltanoAnalysisFileParser.package_files(package)
sortedM5oFiles["topics"]["items"] += package_files["topics"]["items"]
sortedM5oFiles["tables"]["items"] += package_files["tables"]["items"]
if not len(sortedM5oFiles["topics"]["items"]):
return jsonify(
{
"result": False,
"errors": [{"message": "Missing topic file(s)", "file_name": "*"}],
"files": [],
}
)
return jsonify(sortedM5oFiles)
def test_plugin_configuration(plugin_ref) -> Response:
"""
Endpoint for testing a plugin configuration's valid connection
"""
project = Project.find()
payload = request.get_json()
plugins_service = ProjectPluginsService(project)
plugin = plugins_service.get_plugin(plugin_ref)
settings = PluginSettingsService(project,
plugin,
plugins_service=plugins_service,
show_hidden=False)
config = payload.get("config", {})
valid_config = {
name: value
for name, value in config.items()
if validate_plugin_config(plugin, name, value, project, settings)
}
settings.config_override = PluginSettingsService.unredact(valid_config)
async def test_stream(tap_stream) -> bool:
while not tap_stream.at_eof():
message = await tap_stream.readline()
json_dict = json.loads(message)
if json_dict["type"] == "RECORD":
return True
return False
async def test_extractor():
process = None
try:
invoker = invoker_factory(
project,
plugin,
plugins_service=plugins_service,
plugin_settings_service=settings,
)
with invoker.prepared(db.session):
process = await invoker.invoke_async(
stdout=asyncio.subprocess.PIPE)
return await test_stream(process.stdout)
except Exception as err:
logging.debug(err)
# if anything happens, this is not successful
return False
finally:
try:
if process:
psutil.Process(process.pid).terminate()
except Exception as err:
logging.debug(err)
loop = asyncio.get_event_loop()
success = loop.run_until_complete(test_extractor())
return jsonify({"is_success": success}), 200
def download_job_log(job_id) -> Response:
"""
Endpoint for downloading a job log with job_id
"""
project = Project.find()
log_service = JobLoggingService(project)
return send_file(log_service.get_downloadable_log(job_id),
mimetype="text/plain")
def __init__(self):
project = Project.find()
self.settings_file_path = project.root_dir("model",
"database.settings.m5o")
if not self.settings_file_path.is_file():
with open(self.settings_file_path, "w") as f:
settings = {"settings": {"connections": []}}
json.dump(settings, f)
def update_report(self, data):
project = Project.find()
file_path = project.analyze_dir("reports",
f"{data['slug']}.report.m5o")
with open(file_path, "w") as f:
json.dump(data, f)
return data
def test_init(self, cli_runner, tmp_path_factory, pushd):
new_project_root = tmp_path_factory.mktemp("new_meltano_root")
pushd(new_project_root)
# there are no project actually
assert Project._default is None
with pytest.raises(ProjectNotFound):
Project.find()
# create one with the CLI
cli_runner.invoke(cli, ["init", "test_project", "--no_usage_stats"])
pushd("test_project")
project = Project.find()
# Deactivate project
Project.deactivate()
files = (
project.root.joinpath(file).resolve()
for file in ("meltano.yml", "README.md", ".gitignore", "requirements.txt")
)
dirs = (
project.root.joinpath(dir)
for dir in (
"model",
"extract",
"load",
"transform",
"analyze",
"notebook",
"orchestrate",
)
)
for file in files:
assert file.is_file()
for dir in dirs:
assert dir.is_dir()
meltano_yml = project.root_dir("meltano.yml").read_text()
assert "send_anonymous_usage_stats: false" in meltano_yml
assert "project_id:" not in meltano_yml
def test_find(self, project, mkdtemp, monkeypatch):
# defaults to the cwd
found = Project.find(activate=False)
assert found == project
# or you can specify a path
found = Project.find(project.root, activate=False)
assert found == project
# or set the MELTANO_PROJECT_ROOT env var
with monkeypatch.context() as m:
m.chdir(project.root.joinpath("model"))
m.setenv(PROJECT_ROOT_ENV, "..")
found = Project.find(activate=False)
assert found == project
# but it doens't recurse up, you have to be
# at the meltano.yml level
with pytest.raises(ProjectNotFound):
Project.find(project.root.joinpath("model"))
# and it fails if there isn't a meltano.yml
with pytest.raises(ProjectNotFound):
try:
empty_dir = mkdtemp("meltano_empty_project")
Project.find(empty_dir)
finally:
shutil.rmtree(empty_dir)
def get_topic(self, namespace, topic_name):
project = Project.find()
filename = enforce_secure_filename(f"{topic_name}.topic.m5oc")
topic = project.run_dir("models", namespace, filename)
with topic.open() as f:
topic = json.load(f)
return topic
def all():
project = Project.find()
discovery = PluginDiscoveryService(project)
ordered_plugins = {}
for type, plugins in groupby(discovery.plugins(), key=lambda p: p.type):
ordered_plugins[type] = [plugin.canonical() for plugin in plugins]
return jsonify(ordered_plugins)
def __init__(self, project: Project):
self.project = project
self._output_dir = self.project.run_dir("models")
self.topics = []
self.tables = []
self.packaged_topics = []
self.packaged_tables = []
self.required_topic_properties = ["name", "label", "designs"]
self.required_design_properties = ["from", "label", "description"]
self.required_join_properties = ["sql_on", "relationship"]
self.required_table_properties = ["sql_table_name", "columns"]
self.join_relationship_types = [
"one_to_one",
"one_to_many",
"many_to_one",
"many_to_many",
]
self.project = Project()
