def view_all_runs(status, date_from, date_to, pipeline, parent_id, find, top):
runs_table = prettytable.PrettyTable()
runs_table.field_names = ["RunID", "Parent RunID", "Pipeline", "Version", "Status", "Started"]
runs_table.align = "r"
try:
statuses = []
if status is not None:
if status.upper() != 'ANY':
for status_value in status.split(','):
statuses.append(status_value.upper())
else:
statuses.append('RUNNING')
pipeline_id = None
pipeline_version_name = None
if pipeline is not None:
pipeline_name_parts = pipeline.split('@')
pipeline_model = Pipeline.get(pipeline_name_parts[0])
pipeline_id = pipeline_model.identifier
pipeline_version_name = pipeline_model.current_version_name
if len(pipeline_name_parts) > 1:
pipeline_version_name = pipeline_name_parts[1]
page = DEFAULT_PAGE_INDEX
page_size = DEFAULT_PAGE_SIZE
if top is not None:
page = 1
page_size = top
run_filter = PipelineRun.list(page=page,
page_size=page_size,
statuses=statuses,
date_from=date_utilities.parse_date_parameter(date_from),
date_to=date_utilities.parse_date_parameter(date_to),
pipeline_id=pipeline_id,
version=pipeline_version_name,
parent_id=parent_id,
custom_filter=find)
if run_filter.total_count == 0:
click.echo('No data is available for the request')
else:
if run_filter.total_count > run_filter.page_size:
click.echo('Showing {} results from {}:'.format(run_filter.page_size, run_filter.total_count))
for run_model in run_filter.elements:
runs_table.add_row([run_model.identifier,
run_model.parent_id,
run_model.pipeline,
run_model.version,
state_utilities.color_state(run_model.status),
run_model.scheduled_date])
click.echo(runs_table)
click.echo()
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
except requests.exceptions.RequestException as http_error:
click.echo('Http error: {}'.format(str(http_error)), err=True)
except RuntimeError as runtime_error:
click.echo('Error: {}'.format(str(runtime_error)), err=True)
except ValueError as value_error:
click.echo('Error: {}'.format(str(value_error)), err=True)
def prettytable(self):
"""Returns a new prettytable instance."""
table = prettytable.PrettyTable(self.columns)
if self.sortby:
if self.sortby in self.columns:
table.sortby = self.sortby
else:
msg = "Column (%s) doesn't exist to sort by" % self.sortby
raise exceptions.CLIAbort(msg)
for a_col, alignment in self.align.items():
table.align[a_col] = alignment
if self.title:
table.title = self.title
# Adding rows
for row in self.rows:
table.add_row(row)
return table
def view_all_cluster():
nodes_table = prettytable.PrettyTable()
nodes_table.field_names = ["Name", "Pipeline", "Run", "Addresses", "Created"]
nodes_table.align = "l"
try:
nodes = list(Cluster.list())
if len(nodes) > 0:
for node_model in nodes:
info_lines = []
is_first_line = True
pipeline_name = None
run_id = None
if node_model.run is not None:
pipeline_name = node_model.run.pipeline
run_id = node_model.run.identifier
for address in node_model.addresses:
if is_first_line:
info_lines.append([node_model.name, pipeline_name, run_id, address, node_model.created])
else:
info_lines.append(['', '', '', address, ''])
is_first_line = False
if len(info_lines) == 0:
info_lines.append([node_model.name, pipeline_name, run_id, None, node_model.created])
for line in info_lines:
nodes_table.add_row(line)
nodes_table.add_row(['', '', '', '', ''])
click.echo(nodes_table)
else:
click.echo('No data is available for the request')
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
except requests.exceptions.RequestException as http_error:
click.echo('Http error: {}'.format(str(http_error)), err=True)
except RuntimeError as runtime_error:
click.echo('Error: {}'.format(str(runtime_error)), err=True)
except ValueError as value_error:
click.echo('Error: {}'.format(str(value_error)), err=True)
def view_all_pipes():
pipes_table = prettytable.PrettyTable()
pipes_table.field_names = ["ID", "Name", "Latest version", "Created", "Source repo"]
pipes_table.align = "r"
try:
pipelines = list(Pipeline.list())
if len(pipelines) > 0:
for pipeline_model in pipelines:
pipes_table.add_row([pipeline_model.identifier,
pipeline_model.name,
pipeline_model.current_version_name,
pipeline_model.created_date,
pipeline_model.repository])
click.echo(pipes_table)
else:
click.echo('No pipelines are available')
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
except requests.exceptions.RequestException as http_error:
click.echo('Http error: {}'.format(str(http_error)), err=True)
except RuntimeError as runtime_error:
click.echo('Error: {}'.format(str(runtime_error)), err=True)
except ValueError as value_error:
click.echo('Error: {}'.format(str(value_error)), err=True)
def prettytable(self):
"""Returns a new prettytable instance."""
table = prettytable.PrettyTable(self.columns)
if self.sortby:
if self.sortby in self.columns:
table.sortby = self.sortby
else:
msg = "Column (%s) doesn't exist to sort by" % self.sortby
raise exceptions.CLIAbort(msg)
if isinstance(self.align, str):
table.align = self.align
else:
# Required because PrettyTable has a strict setter function for alignment
for a_col, alignment in self.align.items():
table.align[a_col] = alignment
if self.title:
table.title = self.title
# Adding rows
for row in self.rows:
table.add_row(row)
return table
def run(cls, pipeline, config, parameters, yes, run_params, instance_disk, instance_type, docker_image,
cmd_template, timeout, quiet, instance_count, cores, sync, price_type=None):
# All pipeline run parameters can be specified as options, e.g. --read1 /path/to/reads.fastq
# In this case - runs_params_dict will contain keys-values for each option, e.g. {'--read1': '/path/to/reads.fastq'}
# So they can be addressed with run_params_dict['--read1']
# This approach is used because we do not know parameters list beforehand
run_params_dict = dict([(k.strip('-'), v) for k, v in zip(run_params[::2], run_params[1::2])])
# Calculate instance_type and instance_count if only cores specified
if not instance_count and cores:
nodes_spec = ClusterManager.calculate_cluster_from_cores(cores, core_type=instance_type)
instance_count = nodes_spec["count"]
if instance_count > 1:
instance_count -= 1
else:
instance_count = None
instance_type = nodes_spec["name"]
try:
if pipeline:
parts = pipeline.split('@')
pipeline_name = parts[0]
if not quiet:
click.echo('Fetching pipeline info...', nl=False)
pipeline_model = Pipeline.get(
pipeline_name,
load_versions=False,
load_storage_rules=False,
load_run_parameters=len(parts) == 1,
config_name=config)
if not quiet:
click.echo('done.', nl=True)
pipeline_run_parameters = pipeline_model.current_version.run_parameters
if len(parts) > 1:
if not quiet:
click.echo('Fetching parameters...', nl=False)
pipeline_run_parameters = Pipeline.load_run_parameters(pipeline_model.identifier, parts[1],
config_name=config)
if not quiet:
click.echo('done.', nl=True)
if parameters:
cls.print_pipeline_parameters_info(pipeline_model, pipeline_run_parameters)
else:
if not quiet:
click.echo('Evaluating estimated price...', nl=False)
run_price = Pipeline.get_estimated_price(pipeline_model.identifier,
pipeline_run_parameters.version,
instance_type,
instance_disk,
config_name=config,
price_type=price_type)
click.echo('done.', nl=True)
price_table = prettytable.PrettyTable()
price_table.field_names = ["key", "value"]
price_table.align = "l"
price_table.set_style(12)
price_table.header = False
price_table.add_row(['Price per hour ({}, hdd {})'.format(run_price.instance_type,
run_price.instance_disk),
'{} $'.format(round(run_price.price_per_hour, 2))])
if run_price.minimum_time_price is not None and run_price.minimum_time_price > 0:
price_table.add_row(['Minimum price',
'{} $'.format(round(run_price.minimum_time_price, 2))])
if run_price.average_time_price is not None and run_price.average_time_price > 0:
price_table.add_row(['Average price',
'{} $'.format(round(run_price.average_time_price, 2))])
if run_price.maximum_time_price is not None and run_price.maximum_time_price > 0:
price_table.add_row(['Maximum price',
'{} $'.format(round(run_price.maximum_time_price, 2))])
click.echo()
click.echo(price_table)
click.echo()
# Checking if user provided required parameters:
wrong_parameters = False
for parameter in pipeline_run_parameters.parameters:
if parameter.required and not run_params_dict.get(parameter.name) and parameter.value is None:
if not quiet:
click.echo('"{}" parameter is required'.format(parameter.name), err=True)
else:
click.echo(parameter.name)
sys.exit(1)
wrong_parameters = True
elif run_params_dict.get(parameter.name) is not None:
parameter.value = run_params_dict.get(parameter.name)
for user_parameter in run_params_dict.keys():
custom_parameter = True
for parameter in pipeline_run_parameters.parameters:
if parameter.name.lower() == user_parameter.lower():
custom_parameter = False
break
if custom_parameter:
pipeline_run_parameters.parameters.append(PipelineRunParameterModel(user_parameter,
run_params_dict.get(
user_parameter),
None,
False))
if not wrong_parameters:
if not yes:
click.confirm('Are you sure you want to schedule a run of {}?'.format(pipeline), abort=True)
pipeline_run_model = Pipeline.launch_pipeline(pipeline_model.identifier,
pipeline_run_parameters.version,
pipeline_run_parameters.parameters,
instance_disk,
instance_type,
docker_image,
cmd_template,
timeout,
config_name=config,
instance_count=instance_count,
price_type=price_type)
pipeline_run_id = pipeline_run_model.identifier
if not quiet:
click.echo('"{}@{}" pipeline run scheduled with RunId: {}'
.format(pipeline_model.name, pipeline_run_parameters.version, pipeline_run_id))
if sync:
click.echo('Pipeline run {} completed with status {}'
.format(pipeline_run_id, cls.get_pipeline_processed_status(pipeline_run_id)))
else:
click.echo(pipeline_run_id)
if sync:
click.echo(cls.get_pipeline_processed_status(pipeline_run_id))
elif parameters:
if not quiet:
click.echo('You must specify pipeline for listing parameters', err=True)
elif docker_image is None or instance_type is None or instance_disk is None:
if not quiet:
click.echo('Docker image, instance type and instance disk are required parameters '
'if pipeline was not provided.')
else:
required_parameters = []
if docker_image is None:
required_parameters.append('docker_image')
if instance_type is None:
required_parameters.append('instance_type')
if instance_disk is None:
required_parameters.append('instance_disk')
click.echo(', '.join(required_parameters))
sys.exit(1)
else:
if not yes:
click.confirm('Are you sure you want to schedule a run?', abort=True)
pipeline_run_model = Pipeline.launch_command(instance_disk,
instance_type,
docker_image,
cmd_template,
run_params_dict,
timeout,
instance_count=instance_count,
price_type=price_type)
pipeline_run_id = pipeline_run_model.identifier
if not quiet:
click.echo('Pipeline run scheduled with RunId: {}'.format(pipeline_run_id))
if sync:
click.echo('Pipeline run {} completed with status {}'
.format(pipeline_run_id, cls.get_pipeline_processed_status(pipeline_run_id)))
else:
click.echo(pipeline_run_id)
if sync:
click.echo(cls.get_pipeline_processed_status(pipeline_run_id))
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
if quiet:
sys.exit(2)
except requests.exceptions.RequestException as http_error:
if not quiet:
click.echo('Http error: {}'.format(str(http_error)), err=True)
else:
click.echo(str(http_error), err=True)
sys.exit(2)
except RuntimeError as runtime_error:
if not quiet:
click.echo('Error: {}'.format(str(runtime_error)), err=True)
else:
click.echo(str(runtime_error), err=True)
sys.exit(2)
except ValueError as value_error:
if not quiet:
click.echo('Error: {}'.format(str(value_error)), err=True)
else:
click.echo(str(value_error), err=True)
sys.exit(2)
def view_cluster_for_node(node_name):
try:
node_model = Cluster.get(node_name)
node_main_info_table = prettytable.PrettyTable()
node_main_info_table.field_names = ["key", "value"]
node_main_info_table.align = "l"
node_main_info_table.set_style(12)
node_main_info_table.header = False
node_main_info_table.add_row(['Name:', node_model.name])
pipeline_name = None
if node_model.run is not None:
pipeline_name = node_model.run.pipeline
node_main_info_table.add_row(['Pipeline:', pipeline_name])
addresses_string = ''
for address in node_model.addresses:
addresses_string += address + '; '
node_main_info_table.add_row(['Addresses:', addresses_string])
node_main_info_table.add_row(['Created:', node_model.created])
click.echo(node_main_info_table)
click.echo()
if node_model.system_info is not None:
table = prettytable.PrettyTable()
table.field_names = ["key", "value"]
table.align = "l"
table.set_style(12)
table.header = False
for key, value in node_model.system_info:
table.add_row([key, value])
echo_title('System info:')
click.echo(table)
click.echo()
if node_model.labels is not None:
table = prettytable.PrettyTable()
table.field_names = ["key", "value"]
table.align = "l"
table.set_style(12)
table.header = False
for key, value in node_model.labels:
if key.lower() == 'node-role.kubernetes.io/master':
table.add_row([key, click.style(value, fg='blue')])
elif key.lower() == 'kubeadm.alpha.kubernetes.io/role' and value.lower() == 'master':
table.add_row([key, click.style(value, fg='blue')])
elif key.lower() == 'cloud-pipeline/role' and value.lower() == 'edge':
table.add_row([key, click.style(value, fg='blue')])
elif key.lower() == 'runid':
table.add_row([key, click.style(value, fg='green')])
else:
table.add_row([key, value])
echo_title('Labels:')
click.echo(table)
click.echo()
if node_model.allocatable is not None or node_model.capacity is not None:
ac_table = prettytable.PrettyTable()
ac_table.field_names = ["", "Allocatable", "Capacity"]
ac_table.align = "l"
keys = []
for key in node_model.allocatable.keys():
if key not in keys:
keys.append(key)
for key in node_model.capacity.keys():
if key not in keys:
keys.append(key)
for key in keys:
ac_table.add_row([key, node_model.allocatable.get(key, ''), node_model.capacity.get(key, '')])
click.echo(ac_table)
click.echo()
if len(node_model.pods) > 0:
echo_title("Jobs:", line=False)
if len(node_model.pods) > 0:
pods_table = prettytable.PrettyTable()
pods_table.field_names = ["Name", "Namespace", "Status"]
pods_table.align = "l"
for pod in node_model.pods:
pods_table.add_row([pod.name, pod.namespace, state_utilities.color_state(pod.phase)])
click.echo(pods_table)
else:
click.echo('No jobs are available')
click.echo()
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
except requests.exceptions.RequestException as http_error:
click.echo('Http error: {}'.format(str(http_error)), err=True)
except RuntimeError as runtime_error:
click.echo('Error: {}'.format(str(runtime_error)), err=True)
except ValueError as value_error:
click.echo('Error: {}'.format(str(value_error)), err=True)
def view_run(run_id, node_details, parameters_details, tasks_details):
try:
run_model = PipelineRun.get(run_id)
if not run_model.pipeline and run_model.pipeline_id is not None:
pipeline_model = Pipeline.get(run_model.pipeline_id)
if pipeline_model is not None:
run_model.pipeline = pipeline_model.name
run_model_price = PipelineRun.get_estimated_price(run_id)
run_main_info_table = prettytable.PrettyTable()
run_main_info_table.field_names = ["key", "value"]
run_main_info_table.align = "l"
run_main_info_table.set_style(12)
run_main_info_table.header = False
run_main_info_table.add_row(['ID:', run_model.identifier])
run_main_info_table.add_row(['Pipeline:', run_model.pipeline])
run_main_info_table.add_row(['Version:', run_model.version])
if run_model.owner is not None:
run_main_info_table.add_row(['Owner:', run_model.owner])
if run_model.endpoints is not None and len(run_model.endpoints) > 0:
endpoint_index = 0
for endpoint in run_model.endpoints:
if endpoint_index == 0:
run_main_info_table.add_row(['Endpoints:', endpoint])
else:
run_main_info_table.add_row(['', endpoint])
endpoint_index = endpoint_index + 1
if not run_model.scheduled_date:
run_main_info_table.add_row(['Scheduled', 'N/A'])
else:
run_main_info_table.add_row(['Scheduled:', run_model.scheduled_date])
if not run_model.start_date:
run_main_info_table.add_row(['Started', 'N/A'])
else:
run_main_info_table.add_row(['Started:', run_model.start_date])
if not run_model.end_date:
run_main_info_table.add_row(['Completed', 'N/A'])
else:
run_main_info_table.add_row(['Completed:', run_model.end_date])
run_main_info_table.add_row(['Status:', state_utilities.color_state(run_model.status)])
run_main_info_table.add_row(['ParentID:', run_model.parent_id])
if run_model_price.total_price > 0:
run_main_info_table.add_row(['Estimated price:', '{} $'.format(round(run_model_price.total_price, 2))])
else:
run_main_info_table.add_row(['Estimated price:', 'N/A'])
click.echo(run_main_info_table)
click.echo()
if node_details:
node_details_table = prettytable.PrettyTable()
node_details_table.field_names = ["key", "value"]
node_details_table.align = "l"
node_details_table.set_style(12)
node_details_table.header = False
for key, value in run_model.instance:
if key == PriceType.SPOT:
node_details_table.add_row(['price-type', PriceType.SPOT if value else PriceType.ON_DEMAND])
else:
node_details_table.add_row([key, value])
echo_title('Node details:')
click.echo(node_details_table)
click.echo()
if parameters_details:
echo_title('Parameters:')
if len(run_model.parameters) > 0:
for parameter in run_model.parameters:
click.echo('{}={}'.format(parameter.name, parameter.value))
else:
click.echo('No parameters are configured')
click.echo()
if tasks_details:
echo_title('Tasks:', line=False)
if len(run_model.tasks) > 0:
tasks_table = prettytable.PrettyTable()
tasks_table.field_names = ['Task', 'State', 'Scheduled', 'Started', 'Finished']
tasks_table.align = "r"
for task in run_model.tasks:
scheduled = 'N/A'
started = 'N/A'
finished = 'N/A'
if task.created is not None:
scheduled = task.created
if task.started is not None:
started = task.started
if task.finished is not None:
finished = task.finished
tasks_table.add_row(
[task.name, state_utilities.color_state(task.status), scheduled, started, finished])
click.echo(tasks_table)
else:
click.echo('No tasks are available for run')
click.echo()
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
except requests.exceptions.RequestException as http_error:
click.echo('Http error: {}'.format(str(http_error)), err=True)
except RuntimeError as runtime_error:
click.echo('Error: {}'.format(str(runtime_error)), err=True)
except ValueError as value_error:
click.echo('Error: {}'.format(str(value_error)), err=True)
def view_pipe(pipeline, versions, parameters, storage_rules, permissions):
try:
pipeline_model = Pipeline.get(pipeline, storage_rules, versions, parameters)
pipe_table = prettytable.PrettyTable()
pipe_table.field_names = ["key", "value"]
pipe_table.align = "l"
pipe_table.set_style(12)
pipe_table.header = False
pipe_table.add_row(['ID:', pipeline_model.identifier])
pipe_table.add_row(['Name:', pipeline_model.name])
pipe_table.add_row(['Latest version:', pipeline_model.current_version_name])
pipe_table.add_row(['Created:', pipeline_model.created_date])
pipe_table.add_row(['Source repo:', pipeline_model.repository])
pipe_table.add_row(['Description:', pipeline_model.description])
click.echo(pipe_table)
click.echo()
if parameters and pipeline_model.current_version is not None and pipeline_model.current_version.run_parameters is not None:
echo_title('Parameters:', line=False)
if len(pipeline_model.current_version.run_parameters.parameters) > 0:
parameters_table = prettytable.PrettyTable()
parameters_table.field_names = ["Name", "Type", "Mandatory", "Default value"]
parameters_table.align = "l"
for parameter in pipeline_model.current_version.run_parameters.parameters:
parameters_table.add_row(
[parameter.name, parameter.parameter_type, parameter.required, parameter.value])
click.echo(parameters_table)
click.echo()
else:
click.echo('No parameters are available for current version')
if versions:
echo_title('Versions:', line=False)
if len(pipeline_model.versions) > 0:
versions_table = prettytable.PrettyTable()
versions_table.field_names = ["Name", "Created", "Draft"]
versions_table.align = "r"
for version_model in pipeline_model.versions:
versions_table.add_row([version_model.name, version_model.created_date, version_model.draft])
click.echo(versions_table)
click.echo()
else:
click.echo('No versions are configured for pipeline')
if storage_rules:
echo_title('Storage rules', line=False)
if len(pipeline_model.storage_rules) > 0:
storage_rules_table = prettytable.PrettyTable()
storage_rules_table.field_names = ["File mask", "Created", "Move to STS"]
storage_rules_table.align = "r"
for rule in pipeline_model.storage_rules:
storage_rules_table.add_row([rule.file_mask, rule.created_date, rule.move_to_sts])
click.echo(storage_rules_table)
click.echo()
else:
click.echo('No storage rules are configured for pipeline')
if permissions:
permissions_list = User.get_permissions(pipeline_model.identifier, 'pipeline')
echo_title('Permissions', line=False)
if len(permissions_list) > 0:
permissions_table = prettytable.PrettyTable()
permissions_table.field_names = ["SID", "Principal", "Allow", "Deny"]
permissions_table.align = "r"
for permission in permissions_list:
permissions_table.add_row([permission.name,
permission.principal,
permission.get_allowed_permissions_description(),
permission.get_denied_permissions_description()])
click.echo(permissions_table)
click.echo()
else:
click.echo('No user permissions are configured for pipeline')
except ConfigNotFoundError as config_not_found_error:
click.echo(str(config_not_found_error), err=True)
except RuntimeError as error:
click.echo(str(error), err=True)
except requests.exceptions.RequestException as http_error:
click.echo('Http error: {}'.format(str(http_error)), err=True)
except ValueError as value_error:
click.echo('Error: {}'.format(str(value_error)), err=True)
async def massaddchapter(self, ctx: CstmContext, *, flags: MassAddFlags):
"""
Description
==============
Add multiple chapters at once.
After entering the command, you are expected to post links to the raws of the chapters.
Each link must be on its own line.
Required Role
=====================
Role `Neko Workers`.
Example
========
.. image:: /images/bigmac.png
:width: 400
:alt: bigmac Example
Arguments
===========
Required
---------
:chapter:
| Chapter on which to start on. Chapter to end on is determined by amount of links sent. [:doc:`/Types/chapter`]
:project:
| Project the chapters belong to. [:doc:`/Types/project`]
"""
start_chp = flags.chapter
date_created = func.now()
project = flags.project
session = ctx.session
message = await ctx.send(
"Please paste the links, with one link per line.")
def check(message):
return (message.author == ctx.message.author
and message.channel == ctx.channel)
try:
message2 = await self.bot.wait_for("message",
timeout=30.0,
check=check)
except asyncio.TimeoutError:
await message.edit(
content=
"No reaction from command author. Chapters was not added.")
return
else:
content = message2.content.split("\n")
chapters = []
for i, link in enumerate(content, start_chp):
chp = Chapter(i, link)
chp.date_created = date_created
session.add(chp)
chp.project = project
chapters.append(chp)
# prompt user to confirm
table = prettytable.PrettyTable(["Chapter", "Link"])
for chp in chapters:
table.add_row([chp.number, chp.link_raw])
image = await misc.drawimage(table.get_string())
await ctx.prompt_and_commit(
text=
f"Do you really want to add these chapters to project {project.title}?",
file=image,
)
