def elt(project, extractor, loader, dry, transform, job_id):
"""
meltano elt EXTRACTOR_NAME LOADER_NAME
extractor_name: Which extractor should be used in this extraction
loader_name: Which loader should be used in this extraction
"""
job_logging_service = JobLoggingService(project)
job = Job(
job_id=job_id
or f'job_{datetime.datetime.now().strftime("%Y%m%d-%H:%M:%S.%f")}')
_, Session = project_engine(project)
session = Session()
try:
with job.run(session), job_logging_service.create_log(
job.job_id, job.run_id) as log_file, OutputLogger(log_file):
try:
success = install_missing_plugins(project, extractor, loader,
transform)
if not success:
raise click.Abort()
elt_context = (ELTContextBuilder(project).with_job(
job).with_extractor(extractor).with_loader(
loader).with_transform(transform).context(session))
if transform != "only":
run_extract_load(elt_context, session, dry_run=dry)
else:
click.secho("Extract & load skipped.", fg="yellow")
if elt_context.transformer:
# Use a new session for the Transform Part to address the last
# update for Job state not being saved in the DB
transform_session = Session()
try:
run_transform(elt_context,
transform_session,
dry_run=dry)
finally:
transform_session.close()
else:
click.secho("Transformation skipped.", fg="yellow")
except Exception as err:
logging.error(
f"ELT could not complete, an error happened during the process: {err}"
)
raise click.Abort()
finally:
session.close()
# fmt: on
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_elt(extractor=extractor,
loader=loader,
transform=transform)
def elt(
project,
extractor,
loader,
transform,
dry,
full_refresh,
select,
exclude,
catalog,
state,
dump,
job_id,
force,
):
"""
meltano elt EXTRACTOR_NAME LOADER_NAME
extractor_name: Which extractor should be used in this extraction
loader_name: Which loader should be used in this extraction
"""
select_filter = [*select, *(f"!{entity}" for entity in exclude)]
job = Job(
job_id=job_id or
f'{datetime.datetime.utcnow().strftime("%Y-%m-%dT%H%M%S")}--{extractor}--{loader}'
)
_, Session = project_engine(project)
session = Session()
try:
plugins_service = ProjectPluginsService(project)
context_builder = _elt_context_builder(
project,
job,
session,
extractor,
loader,
transform,
dry_run=dry,
full_refresh=full_refresh,
select_filter=select_filter,
catalog=catalog,
state=state,
plugins_service=plugins_service,
)
if dump:
dump_file(context_builder, dump)
else:
run_async(
_run_job(project, job, session, context_builder, force=force))
finally:
session.close()
tracker = GoogleAnalyticsTracker(project)
tracker.track_meltano_elt(extractor=extractor,
loader=loader,
transform=transform)