def pull_pipeline(info, pipeline_id, output_path, no_docs):
"""Copy the configuration of a registered pipeline"""
p_api = ce_api.PipelinesApi(utils.api_client(info))
ws_api = ce_api.WorkspacesApi(utils.api_client(info))
active_user = info[constants.ACTIVE_USER]
ws_id = info[active_user][constants.ACTIVE_WORKSPACE]
all_ps = utils.api_call(
ws_api.get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws_id)
p_uuid = utils.find_closest_uuid(pipeline_id, all_ps)
utils.declare('Pulling pipeline: {}'.format(utils.format_uuid(p_uuid)))
pp = utils.api_call(p_api.get_pipeline_api_v1_pipelines_pipeline_id_get,
pipeline_id=p_uuid)
# Short term fix for these getting into the exp_config
c = pp.pipeline_config
if 'bq_args' in c:
c.pop('bq_args')
if 'ai_platform_training_args' in c:
c.pop('ai_platform_training_args')
utils.save_config(c, output_path, no_docs)
def create_workspace(self, name: Text, provider_id: Text) -> Workspace:
""" Create a new workspace in DB
"""
api = ce_api.WorkspacesApi(self.client)
ws = api_utils.api_call(
func=api.create_workspace_api_v1_workspaces_post,
body=Workspace.creator(name, provider_id))
return Workspace(**ws.to_dict())
def list_pipelines(info, pipeline_id, ignore_empty):
"""List of registered pipelines"""
utils.notice('Fetching pipeline(s). This might take a few seconds... \n')
active_user = info[constants.ACTIVE_USER]
ws = info[active_user][constants.ACTIVE_WORKSPACE]
ws_api = ce_api.WorkspacesApi(utils.api_client(info))
p_api = ce_api.PipelinesApi(utils.api_client(info))
d_api = ce_api.DatasourcesApi(utils.api_client(info))
pipelines = utils.api_call(
ws_api.get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws)
if pipeline_id is not None:
pipeline_id = utils.find_closest_uuid(pipeline_id, pipelines)
pipelines.sort(key=lambda x: x.created_at)
for p in pipelines:
write_check = (len(p.pipeline_runs) > 0 or not ignore_empty) and \
(pipeline_id is None or pipeline_id == p.id)
if write_check:
# THIS WHOLE THING IS HERE FOR A REASON!!!!!!
title = 'PIPELINE NAME: {} PIPELINE ID: {}'.format(
p.name, utils.format_uuid(p.id))
utils.declare(title)
utils.declare('-' * len(title))
if len(p.pipeline_runs) == 0:
click.echo('No runs for this pipeline yet!')
else:
table = []
for r in p.pipeline_runs:
author = utils.api_call(
p_api.get_pipeline_run_user_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_user_get,
p.id,
r.id)
# Resolve datasource
ds_commit = utils.api_call(
d_api.get_single_commit_api_v1_datasources_commits_commit_id_get,
r.datasource_commit_id)
ds = utils.api_call(
d_api.get_datasource_api_v1_datasources_ds_id_get,
ds_commit.datasource_id)
table.append({
'RUN ID': utils.format_uuid(r.id),
'TYPE': r.pipeline_run_type,
'CPUs PER WORKER': r.cpus_per_worker,
'WORKERS': r.workers,
'DATASOURCE': '{}_{}'.format(
ds.name,
utils.format_uuid(r.datasource_commit_id)),
'AUTHOR': author.email,
'CREATED AT': utils.format_date(r.start_time),
})
click.echo(tabulate(table, headers='keys', tablefmt='plain'))
click.echo('\n')
def get_workspaces(self, **kwargs) -> List[Workspace]:
""" Get a list of workspaces
"""
api = ce_api.WorkspacesApi(self.client)
ws_list = api_utils.api_call(
func=api.get_loggedin_workspaces_api_v1_workspaces_get)
workspaces = [Workspace(**ws.to_dict()) for ws in ws_list]
if kwargs:
workspaces = client_utils.filter_objects(workspaces, **kwargs)
return workspaces
def get_pipelines(self, workspace_id: Text, **kwargs) -> List[Pipeline]:
api = ce_api.WorkspacesApi(self.client)
all_ps = api_utils.api_call(
api.
get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
workspace_id)
pipelines = [Pipeline(**p.to_dict()) for p in all_ps]
if kwargs:
pipelines = client_utils.filter_objects(pipelines, **kwargs)
return pipelines
def check_workspace(info):
user = info[constants.ACTIVE_USER]
if constants.ACTIVE_WORKSPACE in info[user]:
ws_api = ce_api.WorkspacesApi(api_client(info))
ws = api_call(ws_api.get_workspace_api_v1_workspaces_workspace_id_get,
info[user][constants.ACTIVE_WORKSPACE])
click.echo('Currently, the active workspace is:')
declare('Workspace Name: {}\n'
'Workspace ID: {}\n'.format(ws.name, format_uuid(ws.id)))
else:
raise click.ClickException(message=error(
"You have not set a workspace to work on.\n"
"'cengine workspace list' to see the possible options \n"
"'cengine workspace set' to select a workspace \n"))
def set_workspace(info, workspace_id):
"""Set workspace to be active"""
user = info[constants.ACTIVE_USER]
api = ce_api.WorkspacesApi(api_client(info))
all_ws = api_call(api.get_loggedin_workspaces_api_v1_workspaces_get)
ws_uuid = find_closest_uuid(workspace_id, all_ws)
api_call(api.get_workspace_api_v1_workspaces_workspace_id_get,
ws_uuid)
info[user][constants.ACTIVE_WORKSPACE] = ws_uuid
info.save()
declare('Active workspace set to id: {id}'.format(id=format_uuid(
ws_uuid)))
def create_workspace(ctx, info, provider_id, name):
"""Create a workspace and set it to be active."""
click.echo('Registering the workspace "{}"...'.format(name))
w_api = ce_api.WorkspacesApi(api_client(info))
p_api = ce_api.ProvidersApi(api_client(info))
p_list = api_call(p_api.get_loggedin_provider_api_v1_providers_get)
p_uuid = find_closest_uuid(provider_id, p_list)
ws = api_call(w_api.create_workspace_api_v1_workspaces_post,
WorkspaceCreate(name=name,
provider_id=p_uuid))
declare('Workspace registered.')
ctx.invoke(set_workspace, workspace_id=ws.id)
def resolve_pipeline_runs(info, source_id, run_type=None):
ws_id = info[info[constants.ACTIVE_USER]][constants.ACTIVE_WORKSPACE]
ws_api = ce_api.WorkspacesApi(api_client(info))
p_api = ce_api.PipelinesApi(api_client(info))
if len(source_id.split(':')) == 2:
pipeline_id, run_id = source_id.split(':')
pipelines = api_call(
ws_api.
get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws_id)
p_id = find_closest_uuid(pipeline_id, pipelines)
runs = api_call(
p_api.get_pipeline_runs_api_v1_pipelines_pipeline_id_runs_get,
p_id)
if run_type:
runs = [r for r in runs if r.pipeline_run_type == run_type]
r_id = find_closest_uuid(run_id, runs)
elif len(source_id.split(':')) == 1:
pipeline_id = source_id
pipelines = api_call(
ws_api.
get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws_id)
p_id = find_closest_uuid(pipeline_id, pipelines)
runs = api_call(
p_api.get_pipeline_runs_api_v1_pipelines_pipeline_id_runs_get,
p_id)
if run_type:
runs = [r for r in runs if r.pipeline_run_type == run_type]
runs.sort(key=lambda x: x.start_time)
if runs:
r_id = runs[-1].id
else:
r_id = None
else:
raise ValueError('Unresolvable pipeline ID')
return p_id, r_id
def list_workspaces(info):
"""List of all workspaces available to the user"""
user = info[constants.ACTIVE_USER]
api = ce_api.WorkspacesApi(api_client(info))
ws_list = api_call(api.get_loggedin_workspaces_api_v1_workspaces_get)
if constants.ACTIVE_WORKSPACE in info[user]:
active_w = info[user][constants.ACTIVE_WORKSPACE]
else:
active_w = None
declare('You have created {count} different '
'workspace(s). \n'.format(count=len(ws_list)))
if ws_list:
table = []
for w in ws_list:
table.append({'Selection': '*' if w.id == active_w else '',
'ID': format_uuid(w.id),
'Name': w.name,
'Provider': format_uuid(w.provider_id)})
click.echo(tabulate(table, headers='keys', tablefmt='presto'))
click.echo()
def get_pipeline_status(info, pipeline_id):
"""Get status of started pipelines"""
utils.notice('Fetching pipeline(s). This might take a few seconds.. \n')
active_user = info[constants.ACTIVE_USER]
ws = info[active_user][constants.ACTIVE_WORKSPACE]
ws_api = ce_api.WorkspacesApi(utils.api_client(info))
p_api = ce_api.PipelinesApi(utils.api_client(info))
d_api = ce_api.DatasourcesApi(utils.api_client(info))
pipelines = utils.api_call(
ws_api.get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
ws)
if pipeline_id is not None:
pipeline_id = utils.find_closest_uuid(pipeline_id, pipelines)
pipelines.sort(key=lambda x: x.created_at)
for p in pipelines:
write_check = (len(p.pipeline_runs) > 0) and \
(pipeline_id is None or pipeline_id == p.id)
if write_check:
title = 'PIPELINE NAME: {} PIPELINE ID: {}'.format(
p.name, utils.format_uuid(p.id))
utils.declare(title)
utils.declare('-' * len(title))
table = []
for r in p.pipeline_runs:
run = utils.api_call(
p_api.get_pipeline_run_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_get,
p.id,
r.id)
# Resolve datasource
ds_commit = utils.api_call(
d_api.get_single_commit_api_v1_datasources_commits_commit_id_get,
r.datasource_commit_id)
ds = utils.api_call(
d_api.get_datasource_api_v1_datasources_ds_id_get,
ds_commit.datasource_id)
if run.end_time:
td = run.end_time - run.start_time
else:
td = datetime.now(timezone.utc) - run.start_time
# # Resolve component status
# stage = utils.get_run_stage(run.pipeline_components)
table.append({
'RUN ID': utils.format_uuid(run.id),
'TYPE': run.pipeline_run_type,
'STATUS': run.status,
# 'STAGE': stage,
'DATASOURCE': '{}_{}'.format(
ds.name, utils.format_uuid(run.datasource_commit_id)),
'DATAPOINTS': '{}'.format(ds_commit.n_datapoints),
# 'RUNNING STAGE': stage,
'START TIME': utils.format_date(run.start_time),
'DURATION': utils.format_timedelta(td),
})
click.echo(tabulate(table, headers='keys', tablefmt='plain'))
click.echo('\n')
def get_pipeline_status(self,
workspace_id: Text,
pipeline_id: Text = None) -> Dict:
ws_api = ce_api.WorkspacesApi(self.client)
p_api = ce_api.PipelinesApi(self.client)
d_api = ce_api.DatasourcesApi(self.client)
status_dict = {}
pipelines = api_utils.api_call(
ws_api.
get_workspaces_pipelines_api_v1_workspaces_workspace_id_pipelines_get,
workspace_id)
pipelines.sort(key=lambda x: x.created_at)
for p in pipelines:
write_check = (len(p.pipeline_runs) > 0) and \
(pipeline_id is None or pipeline_id == p.id)
if write_check:
status_dict[p.id] = []
for r in p.pipeline_runs:
run = api_utils.api_call(
p_api.
get_pipeline_run_api_v1_pipelines_pipeline_id_runs_pipeline_run_id_get,
p.id, r.id)
# Resolve datasource
ds_commit = api_utils.api_call(
d_api.
get_single_commit_api_v1_datasources_commits_commit_id_get,
r.datasource_commit_id)
ds = api_utils.api_call(
d_api.get_datasource_api_v1_datasources_ds_id_get,
ds_commit.datasource_id)
if run.end_time:
td = run.end_time - run.start_time
else:
td = datetime.now(timezone.utc) - run.start_time
status_dict[p.id].append({
'RUN ID':
run.id,
'TYPE':
run.pipeline_run_type,
'STATUS':
run.status,
'DATASOURCE':
'{}_{}'.format(ds.name, run.datasource_commit_id),
'DATAPOINTS':
'{}'.format(ds_commit.n_datapoints),
'START TIME':
print_utils.format_date(run.start_time),
'DURATION':
print_utils.format_timedelta(td),
})
return status_dict