def view_all_runs(status, date_from, date_to, pipeline, parent_id, find, top): runs_table = prettytable.PrettyTable() runs_table.field_names = ["RunID", "Parent RunID", "Pipeline", "Version", "Status", "Started"] runs_table.align = "r" try: statuses = [] if status is not None: if status.upper() != 'ANY': for status_value in status.split(','): statuses.append(status_value.upper()) else: statuses.append('RUNNING') pipeline_id = None pipeline_version_name = None if pipeline is not None: pipeline_name_parts = pipeline.split('@') pipeline_model = Pipeline.get(pipeline_name_parts[0]) pipeline_id = pipeline_model.identifier pipeline_version_name = pipeline_model.current_version_name if len(pipeline_name_parts) > 1: pipeline_version_name = pipeline_name_parts[1] page = DEFAULT_PAGE_INDEX page_size = DEFAULT_PAGE_SIZE if top is not None: page = 1 page_size = top run_filter = PipelineRun.list(page=page, page_size=page_size, statuses=statuses, date_from=date_utilities.parse_date_parameter(date_from), date_to=date_utilities.parse_date_parameter(date_to), pipeline_id=pipeline_id, version=pipeline_version_name, parent_id=parent_id, custom_filter=find) if run_filter.total_count == 0: click.echo('No data is available for the request') else: if run_filter.total_count > run_filter.page_size: click.echo('Showing {} results from {}:'.format(run_filter.page_size, run_filter.total_count)) for run_model in run_filter.elements: runs_table.add_row([run_model.identifier, run_model.parent_id, run_model.pipeline, run_model.version, state_utilities.color_state(run_model.status), run_model.scheduled_date]) click.echo(runs_table) click.echo() except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) except requests.exceptions.RequestException as http_error: click.echo('Http error: {}'.format(str(http_error)), err=True) except RuntimeError as runtime_error: click.echo('Error: {}'.format(str(runtime_error)), err=True) except ValueError as value_error: click.echo('Error: {}'.format(str(value_error)), err=True)
def prettytable(self): """Returns a new prettytable instance.""" table = prettytable.PrettyTable(self.columns) if self.sortby: if self.sortby in self.columns: table.sortby = self.sortby else: msg = "Column (%s) doesn't exist to sort by" % self.sortby raise exceptions.CLIAbort(msg) for a_col, alignment in self.align.items(): table.align[a_col] = alignment if self.title: table.title = self.title # Adding rows for row in self.rows: table.add_row(row) return table
def view_all_cluster(): nodes_table = prettytable.PrettyTable() nodes_table.field_names = ["Name", "Pipeline", "Run", "Addresses", "Created"] nodes_table.align = "l" try: nodes = list(Cluster.list()) if len(nodes) > 0: for node_model in nodes: info_lines = [] is_first_line = True pipeline_name = None run_id = None if node_model.run is not None: pipeline_name = node_model.run.pipeline run_id = node_model.run.identifier for address in node_model.addresses: if is_first_line: info_lines.append([node_model.name, pipeline_name, run_id, address, node_model.created]) else: info_lines.append(['', '', '', address, '']) is_first_line = False if len(info_lines) == 0: info_lines.append([node_model.name, pipeline_name, run_id, None, node_model.created]) for line in info_lines: nodes_table.add_row(line) nodes_table.add_row(['', '', '', '', '']) click.echo(nodes_table) else: click.echo('No data is available for the request') except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) except requests.exceptions.RequestException as http_error: click.echo('Http error: {}'.format(str(http_error)), err=True) except RuntimeError as runtime_error: click.echo('Error: {}'.format(str(runtime_error)), err=True) except ValueError as value_error: click.echo('Error: {}'.format(str(value_error)), err=True)
def view_all_pipes(): pipes_table = prettytable.PrettyTable() pipes_table.field_names = ["ID", "Name", "Latest version", "Created", "Source repo"] pipes_table.align = "r" try: pipelines = list(Pipeline.list()) if len(pipelines) > 0: for pipeline_model in pipelines: pipes_table.add_row([pipeline_model.identifier, pipeline_model.name, pipeline_model.current_version_name, pipeline_model.created_date, pipeline_model.repository]) click.echo(pipes_table) else: click.echo('No pipelines are available') except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) except requests.exceptions.RequestException as http_error: click.echo('Http error: {}'.format(str(http_error)), err=True) except RuntimeError as runtime_error: click.echo('Error: {}'.format(str(runtime_error)), err=True) except ValueError as value_error: click.echo('Error: {}'.format(str(value_error)), err=True)
def prettytable(self): """Returns a new prettytable instance.""" table = prettytable.PrettyTable(self.columns) if self.sortby: if self.sortby in self.columns: table.sortby = self.sortby else: msg = "Column (%s) doesn't exist to sort by" % self.sortby raise exceptions.CLIAbort(msg) if isinstance(self.align, str): table.align = self.align else: # Required because PrettyTable has a strict setter function for alignment for a_col, alignment in self.align.items(): table.align[a_col] = alignment if self.title: table.title = self.title # Adding rows for row in self.rows: table.add_row(row) return table
def run(cls, pipeline, config, parameters, yes, run_params, instance_disk, instance_type, docker_image, cmd_template, timeout, quiet, instance_count, cores, sync, price_type=None): # All pipeline run parameters can be specified as options, e.g. --read1 /path/to/reads.fastq # In this case - runs_params_dict will contain keys-values for each option, e.g. {'--read1': '/path/to/reads.fastq'} # So they can be addressed with run_params_dict['--read1'] # This approach is used because we do not know parameters list beforehand run_params_dict = dict([(k.strip('-'), v) for k, v in zip(run_params[::2], run_params[1::2])]) # Calculate instance_type and instance_count if only cores specified if not instance_count and cores: nodes_spec = ClusterManager.calculate_cluster_from_cores(cores, core_type=instance_type) instance_count = nodes_spec["count"] if instance_count > 1: instance_count -= 1 else: instance_count = None instance_type = nodes_spec["name"] try: if pipeline: parts = pipeline.split('@') pipeline_name = parts[0] if not quiet: click.echo('Fetching pipeline info...', nl=False) pipeline_model = Pipeline.get( pipeline_name, load_versions=False, load_storage_rules=False, load_run_parameters=len(parts) == 1, config_name=config) if not quiet: click.echo('done.', nl=True) pipeline_run_parameters = pipeline_model.current_version.run_parameters if len(parts) > 1: if not quiet: click.echo('Fetching parameters...', nl=False) pipeline_run_parameters = Pipeline.load_run_parameters(pipeline_model.identifier, parts[1], config_name=config) if not quiet: click.echo('done.', nl=True) if parameters: cls.print_pipeline_parameters_info(pipeline_model, pipeline_run_parameters) else: if not quiet: click.echo('Evaluating estimated price...', nl=False) run_price = Pipeline.get_estimated_price(pipeline_model.identifier, pipeline_run_parameters.version, instance_type, instance_disk, config_name=config, price_type=price_type) click.echo('done.', nl=True) price_table = prettytable.PrettyTable() price_table.field_names = ["key", "value"] price_table.align = "l" price_table.set_style(12) price_table.header = False price_table.add_row(['Price per hour ({}, hdd {})'.format(run_price.instance_type, run_price.instance_disk), '{} $'.format(round(run_price.price_per_hour, 2))]) if run_price.minimum_time_price is not None and run_price.minimum_time_price > 0: price_table.add_row(['Minimum price', '{} $'.format(round(run_price.minimum_time_price, 2))]) if run_price.average_time_price is not None and run_price.average_time_price > 0: price_table.add_row(['Average price', '{} $'.format(round(run_price.average_time_price, 2))]) if run_price.maximum_time_price is not None and run_price.maximum_time_price > 0: price_table.add_row(['Maximum price', '{} $'.format(round(run_price.maximum_time_price, 2))]) click.echo() click.echo(price_table) click.echo() # Checking if user provided required parameters: wrong_parameters = False for parameter in pipeline_run_parameters.parameters: if parameter.required and not run_params_dict.get(parameter.name) and parameter.value is None: if not quiet: click.echo('"{}" parameter is required'.format(parameter.name), err=True) else: click.echo(parameter.name) sys.exit(1) wrong_parameters = True elif run_params_dict.get(parameter.name) is not None: parameter.value = run_params_dict.get(parameter.name) for user_parameter in run_params_dict.keys(): custom_parameter = True for parameter in pipeline_run_parameters.parameters: if parameter.name.lower() == user_parameter.lower(): custom_parameter = False break if custom_parameter: pipeline_run_parameters.parameters.append(PipelineRunParameterModel(user_parameter, run_params_dict.get( user_parameter), None, False)) if not wrong_parameters: if not yes: click.confirm('Are you sure you want to schedule a run of {}?'.format(pipeline), abort=True) pipeline_run_model = Pipeline.launch_pipeline(pipeline_model.identifier, pipeline_run_parameters.version, pipeline_run_parameters.parameters, instance_disk, instance_type, docker_image, cmd_template, timeout, config_name=config, instance_count=instance_count, price_type=price_type) pipeline_run_id = pipeline_run_model.identifier if not quiet: click.echo('"{}@{}" pipeline run scheduled with RunId: {}' .format(pipeline_model.name, pipeline_run_parameters.version, pipeline_run_id)) if sync: click.echo('Pipeline run {} completed with status {}' .format(pipeline_run_id, cls.get_pipeline_processed_status(pipeline_run_id))) else: click.echo(pipeline_run_id) if sync: click.echo(cls.get_pipeline_processed_status(pipeline_run_id)) elif parameters: if not quiet: click.echo('You must specify pipeline for listing parameters', err=True) elif docker_image is None or instance_type is None or instance_disk is None: if not quiet: click.echo('Docker image, instance type and instance disk are required parameters ' 'if pipeline was not provided.') else: required_parameters = [] if docker_image is None: required_parameters.append('docker_image') if instance_type is None: required_parameters.append('instance_type') if instance_disk is None: required_parameters.append('instance_disk') click.echo(', '.join(required_parameters)) sys.exit(1) else: if not yes: click.confirm('Are you sure you want to schedule a run?', abort=True) pipeline_run_model = Pipeline.launch_command(instance_disk, instance_type, docker_image, cmd_template, run_params_dict, timeout, instance_count=instance_count, price_type=price_type) pipeline_run_id = pipeline_run_model.identifier if not quiet: click.echo('Pipeline run scheduled with RunId: {}'.format(pipeline_run_id)) if sync: click.echo('Pipeline run {} completed with status {}' .format(pipeline_run_id, cls.get_pipeline_processed_status(pipeline_run_id))) else: click.echo(pipeline_run_id) if sync: click.echo(cls.get_pipeline_processed_status(pipeline_run_id)) except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) if quiet: sys.exit(2) except requests.exceptions.RequestException as http_error: if not quiet: click.echo('Http error: {}'.format(str(http_error)), err=True) else: click.echo(str(http_error), err=True) sys.exit(2) except RuntimeError as runtime_error: if not quiet: click.echo('Error: {}'.format(str(runtime_error)), err=True) else: click.echo(str(runtime_error), err=True) sys.exit(2) except ValueError as value_error: if not quiet: click.echo('Error: {}'.format(str(value_error)), err=True) else: click.echo(str(value_error), err=True) sys.exit(2)
def view_cluster_for_node(node_name): try: node_model = Cluster.get(node_name) node_main_info_table = prettytable.PrettyTable() node_main_info_table.field_names = ["key", "value"] node_main_info_table.align = "l" node_main_info_table.set_style(12) node_main_info_table.header = False node_main_info_table.add_row(['Name:', node_model.name]) pipeline_name = None if node_model.run is not None: pipeline_name = node_model.run.pipeline node_main_info_table.add_row(['Pipeline:', pipeline_name]) addresses_string = '' for address in node_model.addresses: addresses_string += address + '; ' node_main_info_table.add_row(['Addresses:', addresses_string]) node_main_info_table.add_row(['Created:', node_model.created]) click.echo(node_main_info_table) click.echo() if node_model.system_info is not None: table = prettytable.PrettyTable() table.field_names = ["key", "value"] table.align = "l" table.set_style(12) table.header = False for key, value in node_model.system_info: table.add_row([key, value]) echo_title('System info:') click.echo(table) click.echo() if node_model.labels is not None: table = prettytable.PrettyTable() table.field_names = ["key", "value"] table.align = "l" table.set_style(12) table.header = False for key, value in node_model.labels: if key.lower() == 'node-role.kubernetes.io/master': table.add_row([key, click.style(value, fg='blue')]) elif key.lower() == 'kubeadm.alpha.kubernetes.io/role' and value.lower() == 'master': table.add_row([key, click.style(value, fg='blue')]) elif key.lower() == 'cloud-pipeline/role' and value.lower() == 'edge': table.add_row([key, click.style(value, fg='blue')]) elif key.lower() == 'runid': table.add_row([key, click.style(value, fg='green')]) else: table.add_row([key, value]) echo_title('Labels:') click.echo(table) click.echo() if node_model.allocatable is not None or node_model.capacity is not None: ac_table = prettytable.PrettyTable() ac_table.field_names = ["", "Allocatable", "Capacity"] ac_table.align = "l" keys = [] for key in node_model.allocatable.keys(): if key not in keys: keys.append(key) for key in node_model.capacity.keys(): if key not in keys: keys.append(key) for key in keys: ac_table.add_row([key, node_model.allocatable.get(key, ''), node_model.capacity.get(key, '')]) click.echo(ac_table) click.echo() if len(node_model.pods) > 0: echo_title("Jobs:", line=False) if len(node_model.pods) > 0: pods_table = prettytable.PrettyTable() pods_table.field_names = ["Name", "Namespace", "Status"] pods_table.align = "l" for pod in node_model.pods: pods_table.add_row([pod.name, pod.namespace, state_utilities.color_state(pod.phase)]) click.echo(pods_table) else: click.echo('No jobs are available') click.echo() except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) except requests.exceptions.RequestException as http_error: click.echo('Http error: {}'.format(str(http_error)), err=True) except RuntimeError as runtime_error: click.echo('Error: {}'.format(str(runtime_error)), err=True) except ValueError as value_error: click.echo('Error: {}'.format(str(value_error)), err=True)
def view_run(run_id, node_details, parameters_details, tasks_details): try: run_model = PipelineRun.get(run_id) if not run_model.pipeline and run_model.pipeline_id is not None: pipeline_model = Pipeline.get(run_model.pipeline_id) if pipeline_model is not None: run_model.pipeline = pipeline_model.name run_model_price = PipelineRun.get_estimated_price(run_id) run_main_info_table = prettytable.PrettyTable() run_main_info_table.field_names = ["key", "value"] run_main_info_table.align = "l" run_main_info_table.set_style(12) run_main_info_table.header = False run_main_info_table.add_row(['ID:', run_model.identifier]) run_main_info_table.add_row(['Pipeline:', run_model.pipeline]) run_main_info_table.add_row(['Version:', run_model.version]) if run_model.owner is not None: run_main_info_table.add_row(['Owner:', run_model.owner]) if run_model.endpoints is not None and len(run_model.endpoints) > 0: endpoint_index = 0 for endpoint in run_model.endpoints: if endpoint_index == 0: run_main_info_table.add_row(['Endpoints:', endpoint]) else: run_main_info_table.add_row(['', endpoint]) endpoint_index = endpoint_index + 1 if not run_model.scheduled_date: run_main_info_table.add_row(['Scheduled', 'N/A']) else: run_main_info_table.add_row(['Scheduled:', run_model.scheduled_date]) if not run_model.start_date: run_main_info_table.add_row(['Started', 'N/A']) else: run_main_info_table.add_row(['Started:', run_model.start_date]) if not run_model.end_date: run_main_info_table.add_row(['Completed', 'N/A']) else: run_main_info_table.add_row(['Completed:', run_model.end_date]) run_main_info_table.add_row(['Status:', state_utilities.color_state(run_model.status)]) run_main_info_table.add_row(['ParentID:', run_model.parent_id]) if run_model_price.total_price > 0: run_main_info_table.add_row(['Estimated price:', '{} $'.format(round(run_model_price.total_price, 2))]) else: run_main_info_table.add_row(['Estimated price:', 'N/A']) click.echo(run_main_info_table) click.echo() if node_details: node_details_table = prettytable.PrettyTable() node_details_table.field_names = ["key", "value"] node_details_table.align = "l" node_details_table.set_style(12) node_details_table.header = False for key, value in run_model.instance: if key == PriceType.SPOT: node_details_table.add_row(['price-type', PriceType.SPOT if value else PriceType.ON_DEMAND]) else: node_details_table.add_row([key, value]) echo_title('Node details:') click.echo(node_details_table) click.echo() if parameters_details: echo_title('Parameters:') if len(run_model.parameters) > 0: for parameter in run_model.parameters: click.echo('{}={}'.format(parameter.name, parameter.value)) else: click.echo('No parameters are configured') click.echo() if tasks_details: echo_title('Tasks:', line=False) if len(run_model.tasks) > 0: tasks_table = prettytable.PrettyTable() tasks_table.field_names = ['Task', 'State', 'Scheduled', 'Started', 'Finished'] tasks_table.align = "r" for task in run_model.tasks: scheduled = 'N/A' started = 'N/A' finished = 'N/A' if task.created is not None: scheduled = task.created if task.started is not None: started = task.started if task.finished is not None: finished = task.finished tasks_table.add_row( [task.name, state_utilities.color_state(task.status), scheduled, started, finished]) click.echo(tasks_table) else: click.echo('No tasks are available for run') click.echo() except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) except requests.exceptions.RequestException as http_error: click.echo('Http error: {}'.format(str(http_error)), err=True) except RuntimeError as runtime_error: click.echo('Error: {}'.format(str(runtime_error)), err=True) except ValueError as value_error: click.echo('Error: {}'.format(str(value_error)), err=True)
def view_pipe(pipeline, versions, parameters, storage_rules, permissions): try: pipeline_model = Pipeline.get(pipeline, storage_rules, versions, parameters) pipe_table = prettytable.PrettyTable() pipe_table.field_names = ["key", "value"] pipe_table.align = "l" pipe_table.set_style(12) pipe_table.header = False pipe_table.add_row(['ID:', pipeline_model.identifier]) pipe_table.add_row(['Name:', pipeline_model.name]) pipe_table.add_row(['Latest version:', pipeline_model.current_version_name]) pipe_table.add_row(['Created:', pipeline_model.created_date]) pipe_table.add_row(['Source repo:', pipeline_model.repository]) pipe_table.add_row(['Description:', pipeline_model.description]) click.echo(pipe_table) click.echo() if parameters and pipeline_model.current_version is not None and pipeline_model.current_version.run_parameters is not None: echo_title('Parameters:', line=False) if len(pipeline_model.current_version.run_parameters.parameters) > 0: parameters_table = prettytable.PrettyTable() parameters_table.field_names = ["Name", "Type", "Mandatory", "Default value"] parameters_table.align = "l" for parameter in pipeline_model.current_version.run_parameters.parameters: parameters_table.add_row( [parameter.name, parameter.parameter_type, parameter.required, parameter.value]) click.echo(parameters_table) click.echo() else: click.echo('No parameters are available for current version') if versions: echo_title('Versions:', line=False) if len(pipeline_model.versions) > 0: versions_table = prettytable.PrettyTable() versions_table.field_names = ["Name", "Created", "Draft"] versions_table.align = "r" for version_model in pipeline_model.versions: versions_table.add_row([version_model.name, version_model.created_date, version_model.draft]) click.echo(versions_table) click.echo() else: click.echo('No versions are configured for pipeline') if storage_rules: echo_title('Storage rules', line=False) if len(pipeline_model.storage_rules) > 0: storage_rules_table = prettytable.PrettyTable() storage_rules_table.field_names = ["File mask", "Created", "Move to STS"] storage_rules_table.align = "r" for rule in pipeline_model.storage_rules: storage_rules_table.add_row([rule.file_mask, rule.created_date, rule.move_to_sts]) click.echo(storage_rules_table) click.echo() else: click.echo('No storage rules are configured for pipeline') if permissions: permissions_list = User.get_permissions(pipeline_model.identifier, 'pipeline') echo_title('Permissions', line=False) if len(permissions_list) > 0: permissions_table = prettytable.PrettyTable() permissions_table.field_names = ["SID", "Principal", "Allow", "Deny"] permissions_table.align = "r" for permission in permissions_list: permissions_table.add_row([permission.name, permission.principal, permission.get_allowed_permissions_description(), permission.get_denied_permissions_description()]) click.echo(permissions_table) click.echo() else: click.echo('No user permissions are configured for pipeline') except ConfigNotFoundError as config_not_found_error: click.echo(str(config_not_found_error), err=True) except RuntimeError as error: click.echo(str(error), err=True) except requests.exceptions.RequestException as http_error: click.echo('Http error: {}'.format(str(http_error)), err=True) except ValueError as value_error: click.echo('Error: {}'.format(str(value_error)), err=True)
async def massaddchapter(self, ctx: CstmContext, *, flags: MassAddFlags): """ Description ============== Add multiple chapters at once. After entering the command, you are expected to post links to the raws of the chapters. Each link must be on its own line. Required Role ===================== Role `Neko Workers`. Example ======== .. image:: /images/bigmac.png :width: 400 :alt: bigmac Example Arguments =========== Required --------- :chapter: | Chapter on which to start on. Chapter to end on is determined by amount of links sent. [:doc:`/Types/chapter`] :project: | Project the chapters belong to. [:doc:`/Types/project`] """ start_chp = flags.chapter date_created = func.now() project = flags.project session = ctx.session message = await ctx.send( "Please paste the links, with one link per line.") def check(message): return (message.author == ctx.message.author and message.channel == ctx.channel) try: message2 = await self.bot.wait_for("message", timeout=30.0, check=check) except asyncio.TimeoutError: await message.edit( content= "No reaction from command author. Chapters was not added.") return else: content = message2.content.split("\n") chapters = [] for i, link in enumerate(content, start_chp): chp = Chapter(i, link) chp.date_created = date_created session.add(chp) chp.project = project chapters.append(chp) # prompt user to confirm table = prettytable.PrettyTable(["Chapter", "Link"]) for chp in chapters: table.add_row([chp.number, chp.link_raw]) image = await misc.drawimage(table.get_string()) await ctx.prompt_and_commit( text= f"Do you really want to add these chapters to project {project.title}?", file=image, )