def copy_boot_volume_backup(ctx, from_json, wait_for_state, max_wait_seconds, wait_interval_seconds, boot_volume_backup_id, destination_region, display_name, kms_key_id): if isinstance(boot_volume_backup_id, six.string_types) and len( boot_volume_backup_id.strip()) == 0: raise click.UsageError( 'Parameter --boot-volume-backup-id cannot be whitespace or empty string' ) kwargs = {} kwargs['opc_request_id'] = cli_util.use_or_generate_request_id( ctx.obj['request_id']) details = {} details['destinationRegion'] = destination_region if display_name is not None: details['displayName'] = display_name if kms_key_id is not None: details['kmsKeyId'] = kms_key_id client = cli_util.build_client('core', 'blockstorage', ctx) result = client.copy_boot_volume_backup( boot_volume_backup_id=boot_volume_backup_id, copy_boot_volume_backup_details=details, **kwargs) # Newly created Resource will be in a different region from the origin region. # We should build the client for destination region ctx.obj['region'] = destination_region client = cli_util.build_client('core', 'blockstorage', ctx) if wait_for_state: if hasattr(client, 'get_boot_volume_backup') and callable( getattr(client, 'get_boot_volume_backup')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs[ 'max_interval_seconds'] = wait_interval_seconds click.echo( 'Action completed. Waiting until the resource has entered state: {}' .format(wait_for_state), file=sys.stderr) result = oci.wait_until( client, client.get_boot_volume_backup(result.data.id), 'lifecycle_state', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo( 'Failed to wait until the resource entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo( 'Encountered error while waiting for resource to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo( 'Unable to wait for the resource to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def get_instances(endpoint, identifiers=None, verbose=True, **filters): """ Return instances from a list API endpoint. If not `identifiers` and not `filters` retrieves all objects in database. if `identifiers` and `filters`, identifieres might be filtered. Arguments: endpoint (str): endpoint without API base URL (e.g. `analyses`). identifiers (list): List of identifiers. filters (dict): name, value pairs for API filtering. verbose (bool): print to stderr how many instances will be retrieved. Raises: click.UsageError: if string identifier and endpoint not in individuals, samples or workdflows. Returns: list: of types.SimpleNamespace objects loaded with dicts from API. """ check_system_id = endpoint in {"individuals", "samples", "experiments"} check_name = endpoint in {"assemblies", "techniques", "tags"} instances = [] if verbose: count = len(identifiers or []) count += 0 if identifiers else get_instances_count(endpoint, **filters) ids_msg = " at most " if identifiers else " " # ids may be in filters count = f"Retrieving{ids_msg}{count} from {endpoint} API endpoint..." click.echo(count, err=True) if identifiers is None: instances += iterate(endpoint, **filters) else: for chunk in chunks(identifiers or [], 10000): filters["url"] = endpoint primary_keys = set() names = set() ids = set() for i in map(str, chunk): if i.isdigit(): primary_keys.add(i) elif check_name: names.add(i) elif check_system_id: ids.add(i) else: # pragma: no cover msg = f"msg invalid identifier for {endpoint}: {i}" raise click.UsageError(msg) if primary_keys: instances += iterate(**{"pk__in": ",".join(primary_keys), **filters}) if ids: instances += iterate(**{"system_id__in": ",".join(ids), **filters}) if names: instances += iterate(**{"name__in": ",".join(names), **filters}) return isablfy(instances)
def delete( urn: str, force: bool, soft: bool, env: str, platform: str, entity_type: str, query: str, registry_id: str, dry_run: bool, ) -> None: """Delete metadata from datahub using a single urn or a combination of filters""" cli_utils.test_connectivity_complain_exit("delete") # one of urn / platform / env / query must be provided if not urn and not platform and not env and not query and not registry_id: raise click.UsageError( "You must provide either an urn or a platform or an env or a query for me to delete anything" ) # default query is set to "*" if not provided query = "*" if query is None else query if not force and not soft and not dry_run: click.confirm( "This will permanently delete data from DataHub. Do you want to continue?", abort=True, ) if urn: # Single urn based delete session, host = cli_utils.get_session_and_host() entity_type = guess_entity_type(urn=urn) logger.info(f"DataHub configured with {host}") deletion_result: DeletionResult = delete_one_urn_cmd( urn, soft=soft, dry_run=dry_run, entity_type=entity_type, cached_session_host=(session, host), ) if not dry_run: if deletion_result.num_records == 0: click.echo(f"Nothing deleted for {urn}") else: click.echo( f"Successfully deleted {urn}. {deletion_result.num_records} rows deleted" ) elif registry_id: # Registry-id based delete if soft and not dry_run: raise click.UsageError( "Soft-deleting with a registry-id is not yet supported. Try --dry-run to see what you will be deleting, before issuing a hard-delete using the --hard flag" ) deletion_result = delete_for_registry(registry_id=registry_id, soft=soft, dry_run=dry_run) else: # Filter based delete deletion_result = delete_with_filters( env=env, platform=platform, dry_run=dry_run, soft=soft, entity_type=entity_type, search_query=query, force=force, ) if not dry_run: message = "soft delete" if soft else "hard delete" click.echo( f"Took {(deletion_result.end_time_millis-deletion_result.start_time_millis)/1000.0} seconds to {message} {deletion_result.num_records} rows for {deletion_result.num_entities} entities" ) else: click.echo( f"{deletion_result.num_entities} entities with {deletion_result.num_records if deletion_result.num_records != UNKNOWN_NUM_RECORDS else 'unknown'} rows will be affected. Took {(deletion_result.end_time_millis-deletion_result.start_time_millis)/1000.0} seconds to evaluate." ) if deletion_result.sample_records: click.echo( tabulate(deletion_result.sample_records, RUN_TABLE_COLUMNS, tablefmt="grid"))
def parse_args(self, ctx, args): if not args and self.no_args_is_help and not ctx.resilient_parsing: raise click.UsageError(ctx.get_help()) return super().parse_args(ctx, args)
def _validate_target_and_toolchain_args(target: str, toolchain: str) -> None: if not all([toolchain, target]): raise click.UsageError("--toolchain and --mbed-target arguments are required when generating Mbed config!")
def update_data_mask_rule_target_ids_selected_extended(ctx, from_json, force, wait_for_state, max_wait_seconds, wait_interval_seconds, data_mask_rule_id, display_name, compartment_id, iam_group_id, data_mask_categories, data_mask_rule_status, freeform_tags, defined_tags, if_match, target_selected_values): if isinstance(data_mask_rule_id, six.string_types) and len(data_mask_rule_id.strip()) == 0: raise click.UsageError('Parameter --data-mask-rule-id cannot be whitespace or empty string') if not force: if data_mask_categories or freeform_tags or defined_tags: if not click.confirm("WARNING: Updates to data-mask-categories and freeform-tags and defined-tags will replace any existing values. Are you sure you want to continue?"): ctx.abort() kwargs = {} if if_match is not None: kwargs['if_match'] = if_match kwargs['opc_request_id'] = cli_util.use_or_generate_request_id(ctx.obj['request_id']) _details = {} _details['targetSelected'] = {} if display_name is not None: _details['displayName'] = display_name if compartment_id is not None: _details['compartmentId'] = compartment_id if iam_group_id is not None: _details['iamGroupId'] = iam_group_id if data_mask_categories is not None: _details['dataMaskCategories'] = data_mask_categories if data_mask_rule_status is not None: _details['dataMaskRuleStatus'] = data_mask_rule_status if freeform_tags is not None: _details['freeformTags'] = cli_util.parse_json_parameter("freeform_tags", freeform_tags) if defined_tags is not None: _details['definedTags'] = cli_util.parse_json_parameter("defined_tags", defined_tags) if target_selected_values is not None: _details['targetSelected']['values'] = cli_util.parse_json_parameter("target_selected_values", target_selected_values) _details['targetSelected']['kind'] = 'TARGETIDS' client = cli_util.build_client('cloud_guard', 'cloud_guard', ctx) result = client.update_data_mask_rule( data_mask_rule_id=data_mask_rule_id, update_data_mask_rule_details=_details, **kwargs ) if wait_for_state: if hasattr(client, 'get_data_mask_rule') and callable(getattr(client, 'get_data_mask_rule')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs['max_interval_seconds'] = wait_interval_seconds click.echo('Action completed. Waiting until the resource has entered state: {}'.format(wait_for_state), file=sys.stderr) result = oci.wait_until(client, client.get_data_mask_rule(result.data.id), 'lifecycle_state', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo('Failed to wait until the resource entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo('Encountered error while waiting for resource to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo('Unable to wait for the resource to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def cli(entry_filter, challenge_filter, transpose, format, teams, team_solves): """View users and their progress.""" if teams and not team_solves: raise click.UsageError( "--teams and --no-team-solves are mutually exclusive.") with r8.db: all_challenges = r8.db.execute( "SELECT cid, team FROM challenges WHERE t_start < datetime('now') ORDER BY ROWID" ).fetchall() user_info = r8.db.execute( "SELECT uid, tid FROM users NATURAL LEFT JOIN teams ORDER BY users.ROWID" ).fetchall() submissions = r8.db.execute( "SELECT uid, tid, cid FROM submissions NATURAL JOIN flags NATURAL LEFT JOIN teams" ).fetchall() entries = [] # either teams or users team_users = collections.defaultdict(list) for uid, tid in user_info: if teams: entries.append(tid) else: entries.append(uid) team_users[tid].append(uid) # remove duplicate teams entries = list(dict.fromkeys(entries)) if entry_filter: entries = [ entry for entry in entries if any( entry.startswith(x) for x in entry_filter) ] entry_index = {x: i for i, x in enumerate(entries)} challenges = {} for cid, is_team_challenge in all_challenges: if teams and not is_team_challenge: continue if challenge_filter and not any( cid.startswith(c) for c in challenge_filter): continue challenges[cid] = is_team_challenge if format == "table": SOLVED = "OK" NOT_SOLVED = "FAIL" else: SOLVED = "TRUE" NOT_SOLVED = "FALSE" solved = {cid: [NOT_SOLVED] * len(entries) for cid in challenges} if teams: for _, tid, cid in submissions: if cid in challenges and tid in entry_index: solved[cid][entry_index[tid]] = SOLVED else: for uid, tid, cid in submissions: if cid in challenges: if challenges[cid] and team_solves: for uid in team_users[tid]: if uid in entry_index: solved[cid][entry_index[uid]] = SOLVED elif uid in entry_index: solved[cid][entry_index[uid]] = SOLVED if not transpose and teams: header = "Team" elif not transpose and not teams: header = "User" else: header = "Challenge" table_contents = ([[header] + entries] + [[cid] + solved for cid, solved in solved.items()]) if format == "table": for row in table_contents: row[0] = row[0][:22] if not transpose: table_contents = list(zip(*table_contents)) if format == "table": table = texttable.Texttable(shutil.get_terminal_size((0, 0))[0]) table.set_cols_align(["l"] + ["c"] * (len(table_contents[0]) - 1)) table.set_deco(table.BORDER | table.HEADER | table.VLINES) table.add_rows(table_contents) try: table._compute_cols_width() except ValueError: table._max_width = False tbl = table.draw() print( tbl.replace(SOLVED, click.style(SOLVED, fg="green")).replace( NOT_SOLVED, click.style(NOT_SOLVED, fg="red"))) else: for row in table_contents: print(", ".join((x or "").replace(",", ";") for x in row))
def fail(lineno, line, reason): raise click.UsageError( f'{script_file.name}:{lineno}:"{line}": {reason}')
def python(ctx, runnable, gateway, image, env, passenv, kill, download, allow_overwrite, signum, timeout, detach, port, args): """ Run a python script on the gateway """ if not runnable and not kill: raise click.UsageError('Please supply a RUNNABLE or the --kill option') if not allow_overwrite: for filename in download: file = pathlib.Path(filename) if file.exists(): raise click.UsageError( f'File {filename} exists; please rename it or use the --allow-overwrite flag' ) session = ctx.obj.session if gateway is None: gateway = get_default_gateway(ctx) if kill: signum = _get_signal_number(signum) resp = session.kill_python(gateway, None, signum) resp.raise_for_status() return thread = threading.Thread(target=debug_tunnel, args=(ctx, gateway), daemon=True) thread.start() post_data = [ ('image', image), ('stdout_is_stderr', stdout_is_stderr()), ('detach', '1' if detach else '0'), ] post_data.extend(zip(itertools.repeat('args'), args)) post_data.extend(zip(itertools.repeat('env'), env)) lager_process_id = str(uuid.uuid4()) post_data.append(('env', f'LAGER_PROCESS_ID={lager_process_id}')) post_data.extend( zip(itertools.repeat('env'), [f'{name}={os.environ[name]}' for name in passenv])) post_data.extend( zip(itertools.repeat('portforwards'), [json.dumps(p._asdict()) for p in port])) if timeout is not None: post_data.append(('timeout', timeout)) if os.path.isfile(runnable): post_data.append(('script', open(runnable, 'rb'))) elif os.path.isdir(runnable): try: max_content_size = 20_000_000 zipped_folder = zip_dir(runnable, max_content_size=max_content_size) except SizeLimitExceeded: click.secho( f'Folder content exceeds max size of {max_content_size:,} bytes', err=True, fg='red') ctx.exit(1) if len(zipped_folder) > MAX_ZIP_SIZE: click.secho( f'Zipped module content exceeds max size of {MAX_ZIP_SIZE:,} bytes', err=True, fg='red') ctx.exit(1) post_data.append(('module', zipped_folder)) resp = session.run_python(gateway, files=post_data) kill_python = functools.partial(session.kill_python, gateway, lager_process_id) handler = functools.partial(sigint_handler, kill_python) signal.signal(signal.SIGINT, handler) try: for (datatype, content) in stream_python_output(resp): if datatype == StreamDatatypes.EXIT: _do_exit(content, gateway, session, download) elif datatype == StreamDatatypes.STDOUT: click.echo(content, nl=False) elif datatype == StreamDatatypes.STDERR: click.echo(content, nl=False, err=True) elif datatype == StreamDatatypes.OUTPUT: click.echo(content) except OutputFormatNotSupported: click.secho('Response format not supported. Please upgrade lager-cli', fg='red', err=True) sys.exit(1)
def component_elastic_load_balancer_v2(definition, configuration: dict, args: TemplateArguments, info: dict, force, account_info: AccountArguments): lb_name = configuration["Name"] # domains pointing to the load balancer subdomain = '' main_zone = None for name, domain in configuration.get('Domains', {}).items(): name = '{}{}'.format(lb_name, name) domain_name = "{0}.{1}".format(domain["Subdomain"], domain["Zone"]) convert_cname_records_to_alias(domain_name) properties = { "Type": "A", "Name": domain_name, "HostedZoneName": domain["Zone"], "AliasTarget": { "HostedZoneId": { "Fn::GetAtt": [lb_name, "CanonicalHostedZoneID"] }, "DNSName": { "Fn::GetAtt": [lb_name, "DNSName"] } } } definition["Resources"][name] = { "Type": "AWS::Route53::RecordSet", "Properties": properties } if domain["Type"] == "weighted": definition["Resources"][name]["Properties"]['Weight'] = 0 definition["Resources"][name]["Properties"][ 'SetIdentifier'] = "{0}-{1}".format(info["StackName"], info["StackVersion"]) subdomain = domain['Subdomain'] main_zone = domain['Zone'] # type: str target_group_name = lb_name + 'TargetGroup' listeners = configuration.get('Listeners') or get_listeners( lb_name, target_group_name, subdomain, main_zone, configuration, account_info) health_check_protocol = configuration.get('HealthCheckProtocol') or 'HTTP' if health_check_protocol not in ALLOWED_HEALTH_CHECK_PROTOCOLS: raise click.UsageError( 'Protocol "{}" is not supported for LoadBalancer'.format( health_check_protocol)) health_check_path = configuration.get("HealthCheckPath") or '/health' health_check_port = configuration.get( "HealthCheckPort") or configuration["HTTPPort"] if configuration.get('LoadBalancerName'): loadbalancer_name = generate_valid_cloud_name( configuration["LoadBalancerName"], 32) elif configuration.get('NameSuffix'): version = '{}-{}'.format(info["StackVersion"], configuration['NameSuffix']) loadbalancer_name = get_load_balancer_name(info["StackName"], version) del (configuration['NameSuffix']) else: loadbalancer_name = get_load_balancer_name(info["StackName"], info["StackVersion"]) loadbalancer_scheme = configuration.get('Scheme') or 'internal' if loadbalancer_scheme == 'internet-facing': click.secho( 'You are deploying an internet-facing ELB that will be ' 'publicly accessible! You should have OAUTH2 and HTTPS ' 'in place!', bold=True, err=True) if loadbalancer_scheme not in ALLOWED_LOADBALANCER_SCHEMES: raise click.UsageError( 'Scheme "{}" is not supported for LoadBalancer'.format( loadbalancer_scheme)) if loadbalancer_scheme == "internal": loadbalancer_subnet_map = "LoadBalancerInternalSubnets" else: loadbalancer_subnet_map = "LoadBalancerSubnets" vpc_id = configuration.get("VpcId") or account_info.VpcID tags = [ # Tag "Name" { "Key": "Name", "Value": "{0}-{1}".format(info["StackName"], info["StackVersion"]) }, # Tag "StackName" { "Key": "StackName", "Value": info["StackName"], }, # Tag "StackVersion" { "Key": "StackVersion", "Value": info["StackVersion"] } ] # load balancer definition["Resources"][lb_name] = { "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer", "Properties": { 'Name': loadbalancer_name, 'Scheme': loadbalancer_scheme, 'SecurityGroups': resolve_security_groups(configuration["SecurityGroups"], args.region), 'Subnets': { "Fn::FindInMap": [loadbalancer_subnet_map, { "Ref": "AWS::Region" }, "Subnets"] }, 'LoadBalancerAttributes': [{ "Key": "idle_timeout.timeout_seconds", "Value": "60" }], "Tags": tags } } definition["Resources"][target_group_name] = { 'Type': 'AWS::ElasticLoadBalancingV2::TargetGroup', 'Properties': { 'Name': loadbalancer_name, 'HealthCheckIntervalSeconds': '10', 'HealthCheckPath': health_check_path, 'HealthCheckPort': health_check_port, 'HealthCheckProtocol': health_check_protocol, 'HealthCheckTimeoutSeconds': '5', 'HealthyThresholdCount': '2', 'Port': configuration['HTTPPort'], 'Protocol': 'HTTP', 'UnhealthyThresholdCount': '2', 'VpcId': vpc_id, 'Tags': tags, 'TargetGroupAttributes': [{ 'Key': 'deregistration_delay.timeout_seconds', 'Value': '60' }] } } resource_names = set([lb_name, target_group_name]) for i, listener in enumerate(listeners): if i == 0: suffix = '' else: suffix = str(i + 1) resource_name = lb_name + 'Listener' + suffix definition['Resources'][resource_name] = listener resource_names.add(resource_name) for key, val in configuration.items(): # overwrite any specified properties, but only properties which were defined by us already for res in resource_names: if key in definition['Resources'][res][ 'Properties'] and key not in SENZA_PROPERTIES: definition['Resources'][res]['Properties'][key] = val return definition
def _execute_backfill_command_at_location(cli_args, print_fn, instance, repo_location): external_repo = get_external_repository_from_repo_location( repo_location, cli_args.get("repository")) external_pipeline = get_external_pipeline_from_external_repo( external_repo, cli_args.get("pipeline"), ) noprompt = cli_args.get("noprompt") pipeline_partition_set_names = { external_partition_set.name: external_partition_set for external_partition_set in external_repo.get_external_partition_sets() if external_partition_set.pipeline_name == external_pipeline.name } if not pipeline_partition_set_names: raise click.UsageError( "No partition sets found for pipeline `{}`".format( external_pipeline.name)) partition_set_name = cli_args.get("partition_set") if not partition_set_name: if len(pipeline_partition_set_names) == 1: partition_set_name = next(iter( pipeline_partition_set_names.keys())) elif noprompt: raise click.UsageError( "No partition set specified (see option `--partition-set`)") else: partition_set_name = click.prompt( "Select a partition set to use for backfill: {}".format( ", ".join(x for x in pipeline_partition_set_names.keys()))) partition_set = pipeline_partition_set_names.get(partition_set_name) if not partition_set: raise click.UsageError( "No partition set found named `{}`".format(partition_set_name)) mode = partition_set.mode solid_selection = partition_set.solid_selection run_tags = get_tags_from_args(cli_args) repo_handle = RepositoryHandle( repository_name=external_repo.name, repository_location_handle=repo_location.location_handle, ) # Resolve partitions to backfill partition_names_or_error = repo_location.get_external_partition_names( repo_handle, partition_set_name, ) if isinstance(partition_names_or_error, ExternalPartitionExecutionErrorData): raise DagsterBackfillFailedError( "Failure fetching partition names for {partition_set_name}: {error_message}" .format( partition_set_name=partition_set_name, error_message=partition_names_or_error.error.message, ), serialized_error_info=partition_names_or_error.error, ) partition_names = gen_partition_names_from_args( partition_names_or_error.partition_names, cli_args) # Print backfill info print_fn("\n Pipeline: {}".format(external_pipeline.name)) print_fn("Partition set: {}".format(partition_set_name)) print_fn(" Partitions: {}\n".format( print_partition_format(partition_names, indent_level=15))) # Confirm and launch if noprompt or click.confirm( "Do you want to proceed with the backfill ({} partitions)?".format( len(partition_names))): print_fn("Launching runs... ") backfill_id = make_new_backfill_id() backfill_tags = PipelineRun.tags_for_backfill_id(backfill_id) partition_execution_data = repo_location.get_external_partition_set_execution_param_data( repository_handle=repo_handle, partition_set_name=partition_set_name, partition_names=partition_names, ) if isinstance(partition_execution_data, ExternalPartitionExecutionErrorData): return print_fn("Backfill failed: {}".format( partition_execution_data.error)) assert isinstance(partition_execution_data, ExternalPartitionSetExecutionParamData) for partition_data in partition_execution_data.partition_data: run = _create_external_pipeline_run( instance=instance, repo_location=repo_location, external_repo=external_repo, external_pipeline=external_pipeline, run_config=partition_data.run_config, mode=mode, preset=None, tags=merge_dicts( merge_dicts(partition_data.tags, backfill_tags), run_tags), solid_selection=frozenset(solid_selection) if solid_selection else None, ) instance.launch_run(run.run_id, external_pipeline) print_fn("Launched backfill job `{}`".format(backfill_id)) else: print_fn("Aborted!")
def ls(paths, fields=None, format="auto", recursive=False, jobs=6): """List .nwb files and dandisets metadata. """ # TODO: more logical ordering in case of fields = None from .formatter import JSONFormatter, PYOUTFormatter, YAMLFormatter from ..consts import metadata_all_fields # TODO: avoid from ..support.pyout import PYOUT_SHORT_NAMES_rev from ..utils import find_files common_fields = ("path", "size") all_fields = tuple(sorted(set(common_fields + metadata_all_fields))) if fields is not None: if fields.strip() == "": display_known_fields(all_fields) return fields = fields.split(",") # Map possibly present short names back to full names fields = [PYOUT_SHORT_NAMES_rev.get(f.lower(), f) for f in fields] unknown_fields = set(fields).difference(all_fields) if unknown_fields: display_known_fields(all_fields) raise click.UsageError("Following fields are not known: %s" % ", ".join(unknown_fields)) urls = map(is_url, paths) # Actually I do not see why and it could be useful to compare local-vs-remote # if any(urls) and not all(urls): # raise ValueError(f"ATM cannot mix URLs with local paths. Got {paths}") def assets_gen(): for path in paths: if is_url(path): from ..dandiarchive import navigate_url with navigate_url(path) as (client, dandiset, assets): if dandiset: rec = { "path": dandiset.pop("dandiset", {}).get("identifier", "ERR#%s" % id(dandiset)) } # flatten the metadata into record to display # rec.update(dandiset.get('metadata', {})) rec.update(dandiset) yield rec if recursive and assets: yield from assets else: # For now we support only individual files yield path if recursive: yield from find_files(r"\.nwb\Z", path) if format == "auto": format = "yaml" if any(urls) or (len(paths) == 1 and not recursive) else "pyout" if format == "pyout": if fields and fields[0] != "path": # we must always have path - our "id" fields = ["path"] + fields out = PYOUTFormatter(fields=fields, wait_for_top=3, max_workers=jobs) elif format == "json": out = JSONFormatter() elif format == "json_pp": out = JSONFormatter(indent=2) elif format == "yaml": out = YAMLFormatter() else: raise NotImplementedError("Unknown format %s" % format) async_keys = set(all_fields) if fields is not None: async_keys = async_keys.intersection(fields) async_keys = tuple(async_keys.difference(common_fields)) errors = defaultdict(list) # problem: [] paths with out: for asset in assets_gen(): if isinstance(asset, str): # path rec = {} rec["path"] = asset try: if (not fields or "size" in fields) and not op.isdir(asset): rec["size"] = os.stat(asset).st_size if async_keys: cb = get_metadata_ls(asset, async_keys, errors=errors, flatten=format == "pyout") if format == "pyout": rec[async_keys] = cb else: # TODO: parallel execution # For now just call callback and get all the fields cb_res = cb() # TODO: we should stop masking exceptions in get_metadata_ls, # and centralize logic regardless either it is for pyout or not # and do parallelizaion on our end, so at large it is if cb_res is None: raise for k, v in cb_res.items(): rec[k] = v except Exception as exc: _add_exc_error(asset, rec, errors, exc) elif isinstance(asset, dict): # ready record # TODO: harmonization for pyout rec = asset else: raise TypeError(asset) if not rec: errors["Empty record"].append(asset) lgr.debug("Skipping a record for %s since empty", asset) continue out(rec) if errors: lgr.warning( "Failed to operate on some paths (empty records were listed):\n %s", "\n ".join("%s: %d paths" % (k, len(v)) for k, v in errors.items()), )
def cli(user, password, geometry, start, end, uuid, name, download, sentinel, producttype, instrument, cloud, footprints, path, query, url, order_by, limit): """Search for Sentinel products and, optionally, download all the results and/or create a geojson file with the search result footprints. Beyond your Copernicus Open Access Hub user and password, you must pass a geojson file containing the geometry of the area you want to search for or the UUIDs of the products. If you don't specify the start and end dates, it will search in the last 24 hours. """ _set_logger_handler() if user is None or password is None: try: user, password = requests.utils.get_netrc_auth(url) except TypeError: pass if user is None or password is None: raise click.UsageError( 'Missing --user and --password. Please see docs ' 'for environment variables and .netrc support.') api = SentinelAPI(user, password, url) search_kwargs = {} if sentinel and not (producttype or instrument): search_kwargs["platformname"] = "Sentinel-" + sentinel if instrument and not producttype: search_kwargs["instrumentshortname"] = instrument if producttype: search_kwargs["producttype"] = producttype if cloud: if sentinel not in ['2', '3']: logger.error('Cloud cover is only supported for Sentinel 2 and 3.') exit(1) search_kwargs["cloudcoverpercentage"] = (0, cloud) if query is not None: search_kwargs.update((x.split('=') for x in query)) if geometry is not None: search_kwargs['area'] = geojson_to_wkt(read_geojson(geometry)) if uuid is not None: uuid_list = [x.strip() for x in uuid] products = {} for productid in uuid_list: try: products[productid] = api.get_product_odata(productid) except SentinelAPIError as e: if 'Invalid key' in e.msg: logger.error('No product with ID \'%s\' exists on server', productid) exit(1) else: raise elif name is not None: search_kwargs["identifier"] = name[0] if len( name) == 1 else '(' + ' OR '.join(name) + ')' products = api.query(order_by=order_by, limit=limit, **search_kwargs) else: start = start or "19000101" end = end or "NOW" products = api.query(date=(start, end), order_by=order_by, limit=limit, **search_kwargs) if footprints is True: footprints_geojson = api.to_geojson(products) with open(os.path.join(path, "search_footprints.geojson"), "w") as outfile: outfile.write(gj.dumps(footprints_geojson)) if download is True: product_infos, triggered, failed_downloads = api.download_all( products, path) if len(failed_downloads) > 0: with open(os.path.join(path, "corrupt_scenes.txt"), "w") as outfile: for failed_id in failed_downloads: outfile.write("%s : %s\n" % (failed_id, products[failed_id]['title'])) else: for product_id, props in products.items(): if uuid is None: logger.info('Product %s - %s', product_id, props['summary']) else: # querying uuids has no summary key logger.info('Product %s - %s - %s MB', product_id, props['title'], round(int(props['size']) / (1024. * 1024.), 2)) if uuid is None: logger.info('---') logger.info('%s scenes found with a total size of %.2f GB', len(products), api.get_products_size(products))
def fail(msg): if as_json: echo_json({'success': False, 'error': msg}) sys.exit(1) raise click.UsageError('Could not find content file info: %s' % msg)
def new(commit): """ Create new event """ if not commit: print(f"{Fore.LIGHTYELLOW_EX}Date Format: dd-mm-yyyy{Style.RESET_ALL}") print( f"{Fore.MAGENTA}Leave any date field empty for today's date{Style.RESET_ALL}\n" ) while True: title = prompt("title") if not len(title) > 0: print(f"{Fore.RED}Title cannot be left blank{Style.RESET_ALL}") else: break short_description = prompt("short-description") date_created = prompt("date-of-event") base_date = prompt("base-date(Default: date-of-event)") if not short_description: short_description = title try: date_created = parse_date(date_created) base_date = parse_date(base_date) if date_created > datetime.date.today(): raise Exception( "date-of-event cannot be bigger than today's date.") except Exception as e: raise click.UsageError(f"{Fore.RED}{e}{Style.RESET_ALL}") if not base_date: base_date = date_created syntax = "period = (years:int, months:int, days:int), repeat = (int|*)" print( f"\n{Fore.MAGENTA}Date are calculated for time after `base-date`{Style.RESET_ALL}" ) print(Fore.LIGHTYELLOW_EX + "Syntax: ") print(syntax) print("\nExamples:") print( "period = (0,1,0), repeat = 10 # means to remind every month for 10 times" ) print( "period = (0,0,14), repeat = * # means to remind every 14 days (forever)" ) print( "period = (1,0,0), repeat = 1 # means remind after 1 year (1 time only)" ) print(Style.RESET_ALL) print( f"{Fore.RED}Enter q to exit (There should be atleast one reminder){Style.RESET_ALL}\n" ) event = Event(title, short_description, long_description="", date_created=date_created) model_objects = { "event": event, "reminder_dates": reminder(event, base_date) } create_dir_if_not_exists(BASEDIR, "markdown") markdown_file = path_join(BASEDIR, "markdown", "description.md") template_markdown = path_join(BASEDIR, "templates", "markdown.md") with open(markdown_file, "w") as file: # Read from template markdown with open(template_markdown) as template_file: # Write starter template to this file file.write(template_file.read()) pickle_object(model_objects) print(f"{Style.DIM}Created `description.md` file") print("Run following commands to complete event creation:") print(f"Edit the file with suitable description{Style.RESET_ALL}") print(f"\n>> {Fore.LIGHTBLUE_EX}nano {markdown_file}{Style.RESET_ALL}") print( f">> {Fore.LIGHTBLUE_EX}evem event new --commit{Style.RESET_ALL}") else: model_objects = unpickle_object() if not model_objects: raise click.BadOptionUsage( option_name="markdown", message=("Unable to find cached object." " Run command without `-c/--commit` flag first."), ) os.remove(path_join(BASEDIR, "cache", "__object__.cache")) event = model_objects["event"] event.long_description = read_markdown() event.html = parse_markdown(event) session = session_factory() session.add(event) for model in model_objects["reminder_dates"]: session.add(model) session.commit() session.close() print( f"{Style.BRIGHT}{Fore.GREEN}(\u2713){Style.RESET_ALL} Event successfully commited." )
def delete_imported_package(ctx, from_json, wait_for_state, max_wait_seconds, wait_interval_seconds, oda_instance_id, package_id, if_match): if isinstance(oda_instance_id, six.string_types) and len( oda_instance_id.strip()) == 0: raise click.UsageError( 'Parameter --oda-instance-id cannot be whitespace or empty string') if isinstance(package_id, six.string_types) and len( package_id.strip()) == 0: raise click.UsageError( 'Parameter --package-id cannot be whitespace or empty string') kwargs = {} if if_match is not None: kwargs['if_match'] = if_match kwargs['opc_request_id'] = cli_util.use_or_generate_request_id( ctx.obj['request_id']) client = cli_util.build_client('oda', 'odapackage', ctx) result = client.delete_imported_package(oda_instance_id=oda_instance_id, package_id=package_id, **kwargs) if wait_for_state: if hasattr(client, 'get_work_request') and callable( getattr(client, 'get_work_request')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs[ 'max_interval_seconds'] = wait_interval_seconds click.echo( 'Action completed. Waiting until the work request has entered state: {}' .format(wait_for_state), file=sys.stderr) result = oci.wait_until( client, client.get_work_request( result.headers['opc-work-request-id']), 'status', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo( 'Failed to wait until the work request entered the specified state. Please retrieve the work request to find its current state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo( 'Encountered error while waiting for work request to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo( 'Unable to wait for the work request to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def main(source, repo_source_files, requirements_file, local_source_file, work_dir, python): """ Bundle up a deployment package for AWS Lambda. From your local filesystem: \b $ make-lambda-package . ... dist/lambda-package.zip Or from a remote git repository: \b $ make-lambda-package https://github.com/NoRedInk/make-lambda-package.git ... vendor/dist/NoRedInk-make-lambda-package.zip Use # fragment to specify a commit or a branch: \b $ make-lambda-package https://github.com/NoRedInk/make-lambda-package.git#v1.0.0 Dependencies specified with --requirements-file will built using a docker container that replicates AWS Lambda's execution environment, so that extension modules are correctly packaged. When packaging a local source, --work-dir defaults to `.`: \b * ./build will hold a virtualenv for building dependencies if specified. * ./dist is where the zipped package will be saved When packaging a remote source, --work-dir defaults to `./vendor`. """ scm_source = fsutil.parse_path_or_url(source) paths = fsutil.decide_paths(scm_source, work_dir) if requirements_file: with open(os.devnull, 'w') as devnull: docker_retcode = subprocess.call(['docker', '--help'], stdout=devnull) if docker_retcode != 0: raise click.UsageError( "`docker` command doesn't seem to be available. " "It's required to package dependencies.") if not (requirements_file or repo_source_files or local_source_file): click.secho( 'Warning: without --repo-source-files, --requirements-file, ' 'or --local-source-file, nothing will be included in the zip file. ' 'Assuming you have good reasons to do this and proceeding.', fg='yellow') fsutil.ensure_dirs(paths) if isinstance(scm_source, fsutil.RemoteSource): click.echo('Fetching repo..') scm.fetch_repo(scm_source.url, scm_source.ref, paths.src_dir) deps_file = None if requirements_file: click.echo('Building deps..') deps_file = deps.build_deps(paths, requirements_file, python) click.echo('Creating zip file..') archive.make_archive(paths, repo_source_files, local_source_file, deps_file, python) click.echo(os.path.relpath(paths.zip_path, os.getcwd()))
def update_imported_package(ctx, from_json, force, wait_for_state, max_wait_seconds, wait_interval_seconds, current_package_id, parameter_values, oda_instance_id, package_id, freeform_tags, defined_tags, is_replace_skills, if_match): if isinstance(oda_instance_id, six.string_types) and len( oda_instance_id.strip()) == 0: raise click.UsageError( 'Parameter --oda-instance-id cannot be whitespace or empty string') if isinstance(package_id, six.string_types) and len( package_id.strip()) == 0: raise click.UsageError( 'Parameter --package-id cannot be whitespace or empty string') if not force: if parameter_values or freeform_tags or defined_tags: if not click.confirm( "WARNING: Updates to parameter-values and freeform-tags and defined-tags will replace any existing values. Are you sure you want to continue?" ): ctx.abort() kwargs = {} if is_replace_skills is not None: kwargs['is_replace_skills'] = is_replace_skills if if_match is not None: kwargs['if_match'] = if_match kwargs['opc_request_id'] = cli_util.use_or_generate_request_id( ctx.obj['request_id']) _details = {} _details['currentPackageId'] = current_package_id _details['parameterValues'] = cli_util.parse_json_parameter( "parameter_values", parameter_values) if freeform_tags is not None: _details['freeformTags'] = cli_util.parse_json_parameter( "freeform_tags", freeform_tags) if defined_tags is not None: _details['definedTags'] = cli_util.parse_json_parameter( "defined_tags", defined_tags) client = cli_util.build_client('oda', 'odapackage', ctx) result = client.update_imported_package( oda_instance_id=oda_instance_id, package_id=package_id, update_imported_package_details=_details, **kwargs) if wait_for_state: if hasattr(client, 'get_work_request') and callable( getattr(client, 'get_work_request')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs[ 'max_interval_seconds'] = wait_interval_seconds click.echo( 'Action completed. Waiting until the work request has entered state: {}' .format(wait_for_state), file=sys.stderr) result = oci.wait_until( client, client.get_work_request( result.headers['opc-work-request-id']), 'status', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo( 'Failed to wait until the work request entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo( 'Encountered error while waiting for work request to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo( 'Unable to wait for the work request to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def rename(config, copy, canonical, debug, dry_run, # pylint: disable-msg=too-many-arguments episode, ignore_filelist, log_file, # pylint: disable-msg=too-many-arguments log_level, name, no_cache, output_format, # pylint: disable-msg=too-many-arguments organise, partial, quiet, recursive, # pylint: disable-msg=too-many-arguments rename_dir, regex, season, show, # pylint: disable-msg=too-many-arguments show_override, specials, symlink, the, # pylint: disable-msg=too-many-arguments paths): # pylint: disable-msg=too-many-arguments if debug: log_level = 10 start_logging(log_file, log_level, quiet) logger = functools.partial(log.log, 26) if dry_run or debug: start_dry_run(logger) if copy and symlink: raise click.UsageError("You can't use --copy and --symlink at the same time.") if not paths: paths = [os.getcwd()] for current_dir, filename in build_file_list(paths, recursive, ignore_filelist): try: tv = TvRenamr(current_dir, debug, dry_run, no_cache) _file = File(**tv.extract_details_from_file( filename, user_regex=regex, partial=partial, )) # TODO: Warn setting season & episode will override *all* episodes _file.user_overrides(show, season, episode) _file.safety_check() conf = get_config(config) for ep in _file.episodes: canonical = conf.get( 'canonical', _file.show_name, default=ep.file_.show_name, override=canonical ) # TODO: Warn setting name will override *all* episodes ep.title = tv.retrieve_episode_title( ep, canonical=canonical, override=name, ) # TODO: make this a sanitisation method on ep? ep.title = ep.title.replace('/', '-') show = conf.get_output(_file.show_name, override=show_override) the = conf.get('the', show=_file.show_name, override=the) _file.show_name = tv.format_show_name(show, the=the) _file.set_output_format(conf.get( 'format', _file.show_name, default=_file.output_format, override=output_format )) copy = conf.get( 'copy', _file.show_name, default=False, override=copy ) organise = conf.get( 'organise', _file.show_name, default=True, override=organise ) rename_dir = conf.get( 'renamed', _file.show_name, default=current_dir, override=rename_dir ) specials_folder = conf.get( 'specials_folder', _file.show_name, default='Season 0', override=specials, ) symlink = conf.get( 'symlink', _file.show_name, default=False, override=symlink ) path = tv.build_path( _file, rename_dir=rename_dir, organise=organise, specials_folder=specials_folder, ) tv.rename(filename, path, copy, symlink) except errors.NetworkException: if dry_run or debug: stop_dry_run(logger) sys.exit(1) except (AttributeError, errors.EmptyEpisodeTitleException, errors.EpisodeNotFoundException, errors.IncorrectRegExpException, errors.InvalidXMLException, errors.MissingInformationException, errors.OutputFormatMissingSyntaxException, errors.PathExistsException, errors.ShowNotFoundException, errors.UnexpectedFormatException) as e: continue except Exception as e: if debug: # In debug mode, show the full traceback. raise for msg in e.args: log.critical('Error: %s', msg) sys.exit(1) # if we're not doing a dry run add a blank line for clarity if not (debug and dry_run): log.info('') if dry_run or debug: stop_dry_run(logger)
def update_index(ctx, package_name, force, output_board_index, output_board_dir): """Update board package in the published index. This command will archive and compress a board package and add it to the board index file. A sanity check will be done to ensure the package has a later version than currently in the board index, however this can be disabled with the --force option. The command takes one argument, the name of the board package to update. This should be the name of the package as defined in the board package config INI file section name (use the check_updates command to list all the packages from the config if unsure). """ ctx.obj.load_data() # Load all the package config & metadata. # Use the input board index as the output if none is specified. if output_board_index is None: output_board_index = ctx.obj.board_index_file # Validate that the specified package exists in the config. package = ctx.obj.board_config.get_package(package_name) if package is None: raise click.BadParameter( 'Could not find specified package in the board package config INI file! Run check_updates command to list all configured package names.', param_hint='package') # If not in force mode do a sanity check to make sure the package source # has a newer version than in the index. if not force: # Get all the associated packages in the board index. index_packages = list( ctx.obj.board_index.get_platforms(package.get_parent(), package.get_name())) # Do version check if packages were found in the index. if len(index_packages) > 0: # Find the most recent version in the index packages. latest = max( map(lambda x: parse_version(x.get('version', '')), index_packages)) # Warn if the latest published package is the same or newer than the # current package from its origin source. if latest >= parse_version(package.get_version()): raise click.UsageError( 'Specified package is older than the version currently in the index! Use the --force option to force this update if necessary.' ) # Create the output directory if it doesn't exist. if not os.path.exists(output_board_dir): os.makedirs(output_board_dir) # Build the archive with the board package data and write it to the target # directory. archive_path = os.path.join(output_board_dir, package.get_archive_name()) size, sha256 = package.write_archive(archive_path) click.echo('Created board package archive: {0}'.format(archive_path)) # Convert the package template from JSON to a platform metadata dict that # can be inserted in the board index. template_params = { 'version': package.get_version(), 'filename': package.get_archive_name(), 'sha256': sha256, 'size': size } platform = json.loads(package.get_template().format(**template_params)) # Add the new pacakge metadata to the board index. ctx.obj.board_index.add_platform(package.get_parent(), platform) # Write out the new board index JSON. new_index = ctx.obj.board_index.write_json() with open(output_board_index, 'w') as bi: bi.write(new_index) click.echo( 'Wrote updated board index JSON: {0}'.format(output_board_index))
def show_help(ctx, param, value): if value and not ctx.resilient_parsing: raise click.UsageError(ctx.get_help())
def create_transfer_appliance(ctx, from_json, wait_for_state, max_wait_seconds, wait_interval_seconds, id, customer_shipping_address, minimum_storage_capacity_in_terabytes): if isinstance(id, six.string_types) and len(id.strip()) == 0: raise click.UsageError( 'Parameter --id cannot be whitespace or empty string') kwargs = {} _details = {} if customer_shipping_address is not None: _details['customerShippingAddress'] = cli_util.parse_json_parameter( "customer_shipping_address", customer_shipping_address) if minimum_storage_capacity_in_terabytes is not None: _details[ 'minimumStorageCapacityInTerabytes'] = minimum_storage_capacity_in_terabytes client = cli_util.build_client('dts', 'transfer_appliance', ctx) result = client.create_transfer_appliance( id=id, create_transfer_appliance_details=_details, **kwargs) if wait_for_state: if hasattr(client, 'get_transfer_appliance') and callable( getattr(client, 'get_transfer_appliance')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs[ 'max_interval_seconds'] = wait_interval_seconds click.echo( 'Action completed. Waiting until the resource has entered state: {}' .format(wait_for_state), file=sys.stderr) result = oci.wait_until( client, client.get_transfer_appliance(result.data.id), 'lifecycle_state', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo( 'Failed to wait until the resource entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo( 'Encountered error while waiting for resource to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo( 'Unable to wait for the resource to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def _range_check(p): try: p = int(p) except ValueError: raise click.UsageError("{} is not an int".format(p))
def update_transfer_appliance(ctx, from_json, force, wait_for_state, max_wait_seconds, wait_interval_seconds, id, transfer_appliance_label, lifecycle_state, customer_shipping_address, expected_return_date, pickup_window_start_time, pickup_window_end_time, minimum_storage_capacity_in_terabytes, if_match): if isinstance(id, six.string_types) and len(id.strip()) == 0: raise click.UsageError( 'Parameter --id cannot be whitespace or empty string') if isinstance(transfer_appliance_label, six.string_types) and len( transfer_appliance_label.strip()) == 0: raise click.UsageError( 'Parameter --transfer-appliance-label cannot be whitespace or empty string' ) if not force: if customer_shipping_address: if not click.confirm( "WARNING: Updates to customer-shipping-address will replace any existing values. Are you sure you want to continue?" ): ctx.abort() kwargs = {} if if_match is not None: kwargs['if_match'] = if_match _details = {} if lifecycle_state is not None: _details['lifecycleState'] = lifecycle_state if customer_shipping_address is not None: _details['customerShippingAddress'] = cli_util.parse_json_parameter( "customer_shipping_address", customer_shipping_address) if expected_return_date is not None: _details['expectedReturnDate'] = expected_return_date if pickup_window_start_time is not None: _details['pickupWindowStartTime'] = pickup_window_start_time if pickup_window_end_time is not None: _details['pickupWindowEndTime'] = pickup_window_end_time if minimum_storage_capacity_in_terabytes is not None: _details[ 'minimumStorageCapacityInTerabytes'] = minimum_storage_capacity_in_terabytes client = cli_util.build_client('dts', 'transfer_appliance', ctx) result = client.update_transfer_appliance( id=id, transfer_appliance_label=transfer_appliance_label, update_transfer_appliance_details=_details, **kwargs) if wait_for_state: if hasattr(client, 'get_transfer_appliance') and callable( getattr(client, 'get_transfer_appliance')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs[ 'max_interval_seconds'] = wait_interval_seconds click.echo( 'Action completed. Waiting until the resource has entered state: {}' .format(wait_for_state), file=sys.stderr) result = oci.wait_until( client, client.get_transfer_appliance(result.data.id), 'lifecycle_state', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo( 'Failed to wait until the resource entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo( 'Encountered error while waiting for resource to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo( 'Unable to wait for the resource to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def component_auto_scaling_group(definition, configuration, args, info, force, account_info): definition = ensure_keys(definition, "Resources") # launch configuration config_name = configuration["Name"] + "Config" definition["Resources"][config_name] = { "Type": "AWS::AutoScaling::LaunchConfiguration", "Properties": { "InstanceType": configuration["InstanceType"], "ImageId": { "Fn::FindInMap": ["Images", { "Ref": "AWS::Region" }, configuration["Image"]] }, "AssociatePublicIpAddress": configuration.get('AssociatePublicIpAddress', False), "EbsOptimized": configuration.get('EbsOptimized', False) } } if 'IamRoles' in configuration: logical_id = configuration['Name'] + 'InstanceProfile' roles = configuration['IamRoles'] if len(roles) > 1: for role in roles: if isinstance(role, dict): raise click.UsageError( 'Cannot merge policies of Cloud Formation references ({"Ref": ".."}): ' + 'You can use at most one IAM role with "Ref".') logical_role_id = configuration['Name'] + 'Role' definition['Resources'][logical_role_id] = { 'Type': 'AWS::IAM::Role', 'Properties': { "AssumeRolePolicyDocument": { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Principal": { "Service": ["ec2.amazonaws.com"] }, "Action": ["sts:AssumeRole"] }] }, 'Path': '/', 'Policies': get_merged_policies(roles) } } instance_profile_roles = [{'Ref': logical_role_id}] elif isinstance(roles[0], dict): instance_profile_roles = [ resolve_referenced_resource(roles[0], args.region) ] else: instance_profile_roles = roles definition['Resources'][logical_id] = { 'Type': 'AWS::IAM::InstanceProfile', 'Properties': { 'Path': '/', 'Roles': instance_profile_roles } } definition["Resources"][config_name]["Properties"][ "IamInstanceProfile"] = { 'Ref': logical_id } if "SecurityGroups" in configuration: definition["Resources"][config_name]["Properties"]["SecurityGroups"] = \ resolve_security_groups(configuration["SecurityGroups"], args.region) if "UserData" in configuration: definition["Resources"][config_name]["Properties"]["UserData"] = { "Fn::Base64": configuration["UserData"] } # auto scaling group asg_name = configuration["Name"] asg_success = ["1", "PT15M"] if "AutoScaling" in configuration: if "SuccessRequires" in configuration["AutoScaling"]: asg_success = normalize_asg_success( configuration["AutoScaling"]["SuccessRequires"]) tags = [ # Tag "Name" { "Key": "Name", "PropagateAtLaunch": True, "Value": "{0}-{1}".format(info["StackName"], info["StackVersion"]) }, # Tag "StackName" { "Key": "StackName", "PropagateAtLaunch": True, "Value": info["StackName"], }, # Tag "StackVersion" { "Key": "StackVersion", "PropagateAtLaunch": True, "Value": info["StackVersion"] } ] if "Tags" in configuration: for tag in configuration["Tags"]: tags.append({ "Key": tag["Key"], "PropagateAtLaunch": True, "Value": tag["Value"] }) definition["Resources"][asg_name] = { "Type": "AWS::AutoScaling::AutoScalingGroup", # wait to get a signal from an amount of servers to signal that it booted "CreationPolicy": { "ResourceSignal": { "Count": asg_success[0], "Timeout": asg_success[1] } }, "Properties": { # for our operator some notifications "LaunchConfigurationName": { "Ref": config_name }, "VPCZoneIdentifier": { "Fn::FindInMap": ["ServerSubnets", { "Ref": "AWS::Region" }, "Subnets"] }, "Tags": tags } } asg_properties = definition["Resources"][asg_name]["Properties"] if "OperatorTopicId" in info: asg_properties["NotificationConfiguration"] = { "NotificationTypes": [ "autoscaling:EC2_INSTANCE_LAUNCH", "autoscaling:EC2_INSTANCE_LAUNCH_ERROR", "autoscaling:EC2_INSTANCE_TERMINATE", "autoscaling:EC2_INSTANCE_TERMINATE_ERROR" ], "TopicARN": resolve_topic_arn(args.region, info["OperatorTopicId"]) } default_health_check_type = 'EC2' if "ElasticLoadBalancer" in configuration: if isinstance(configuration["ElasticLoadBalancer"], str): asg_properties["LoadBalancerNames"] = [{ "Ref": configuration["ElasticLoadBalancer"] }] elif isinstance(configuration["ElasticLoadBalancer"], list): asg_properties["LoadBalancerNames"] = [{ 'Ref': ref } for ref in configuration["ElasticLoadBalancer"]] # use ELB health check by default default_health_check_type = 'ELB' if "ElasticLoadBalancerV2" in configuration: if isinstance(configuration["ElasticLoadBalancerV2"], str): asg_properties["TargetGroupARNs"] = [{ "Ref": configuration["ElasticLoadBalancerV2"] + 'TargetGroup' }] elif isinstance(configuration["ElasticLoadBalancerV2"], list): asg_properties["TargetGroupARNs"] = [{ 'Ref': ref } for ref in configuration["ElasticLoadBalancerV2"] + 'TargetGroup' ] # use ELB health check by default default_health_check_type = 'ELB' asg_properties['HealthCheckType'] = configuration.get( 'HealthCheckType', default_health_check_type) asg_properties['HealthCheckGracePeriod'] = configuration.get( 'HealthCheckGracePeriod', 300) if "AutoScaling" in configuration: as_conf = configuration["AutoScaling"] asg_properties["MaxSize"] = as_conf["Maximum"] asg_properties["MinSize"] = as_conf["Minimum"] asg_properties["DesiredCapacity"] = max( int(as_conf["Minimum"]), int(as_conf.get('DesiredCapacity', 1))) default_scaling_adjustment = as_conf.get("ScalingAdjustment", 1) default_cooldown = as_conf.get("Cooldown", "60") # ScaleUp policy scale_up_name = asg_name + "ScaleUp" scale_up_adjustment = int( as_conf.get("ScaleUpAdjustment", default_scaling_adjustment)) scale_up_cooldown = as_conf.get("ScaleUpCooldown", default_cooldown) definition["Resources"][scale_up_name] = create_autoscaling_policy( asg_name, scale_up_name, scale_up_adjustment, scale_up_cooldown, definition) # ScaleDown policy scale_down_name = asg_name + "ScaleDown" scale_down_adjustment = (-1) * int( as_conf.get("ScaleDownAdjustment", default_scaling_adjustment)) scale_down_cooldown = as_conf.get("ScaleDownCooldown", default_cooldown) definition["Resources"][scale_down_name] = create_autoscaling_policy( asg_name, scale_down_name, scale_down_adjustment, scale_down_cooldown, definition) if "MetricType" in as_conf: metric_type = as_conf["MetricType"] metricfns = { "CPU": metric_cpu, "NetworkIn": metric_network, "NetworkOut": metric_network } # lowercase cpu is an acceptable metric, be compatible if metric_type.lower() not in map(lambda t: t.lower(), metricfns.keys()): raise click.UsageError( 'Auto scaling MetricType "{}" not supported.'.format( metric_type)) metricfn = metricfns[metric_type] definition = metricfn(asg_name, definition, as_conf, args, info, force) else: asg_properties["MaxSize"] = 1 asg_properties["MinSize"] = 1 for res in (config_name, asg_name): props = definition['Resources'][res]['Properties'] additional_cf_properties = ADDITIONAL_PROPERTIES.get( definition['Resources'][res]['Type']) properties_allowed_to_overwrite = ( set(props.keys()) - SENZA_PROPERTIES) | additional_cf_properties for key in properties_allowed_to_overwrite: if key in configuration: props[key] = configuration[key] return definition
def cmd( ctx, project, quiet, nix_shell, taskcluster_secrets, taskcluster_client_id, taskcluster_access_token, ): project_config = please_cli.config.PROJECTS.get(project, {}) run_type = project_config.get('run') run_options = project_config.get('run_options', {}) if not run_type: raise click.ClickException( 'Application `{}` is not configured to be runnable.') host = run_options.get('host', 'localhost') if please_cli.config.IN_DOCKER: host = run_options.get('host', '0.0.0.0') port = str(run_options.get('port', 8000)) schema = 'https://' project_name = project.replace('-', '_') ca_cert_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'ca.crt') server_cert_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'server.crt') server_key_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'server.key') os.environ['DEBUG'] = 'true' os.environ['PROJECT_NAME'] = project_name pg_host = please_cli.config.PROJECTS['postgresql']['run_options'].get( 'host', host) pg_port = str( please_cli.config.PROJECTS['postgresql']['run_options']['port']) if 'postgresql' in project_config.get('requires', []): dbname = 'services' click.echo(' => Checking if database `{}` exists ... '.format(dbname), nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke( please_cli.shell.cmd, project=project, quiet=True, command=' '.join([ 'psql', '-lqt', '-h', pg_host, '-p', pg_port, ]), nix_shell=nix_shell, ) database_exists = False for line in output.split('\n'): column1 = line.split('|')[0].strip() if column1 == dbname: database_exists = True break if result != 0: click.secho('ERROR', fg='red') raise click.UsageError('Could not connect to the database.\n\n' 'Please run:\n\n' ' ./please run postgresql\n\n' 'in a separate terminal.') please_cli.utils.check_result(result, output) if not database_exists: click.echo(' => Creating `{}` database ` ... '.format(dbname), nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke( please_cli.shell.cmd, project=project, command=' '.join([ 'createdb', '-h', pg_host, '-p', pg_port, dbname, ]), nix_shell=nix_shell, ) please_cli.utils.check_result(result, output) os.environ['DATABASE_URL'] = 'postgresql://{}:{}/{}'.format( pg_host, pg_port, dbname) if 'redis' in project_config.get('requires', []): # TODO: Support checking if redis is running and support starting redis using please. os.environ['REDIS_URL'] = 'redis://localhost:6379' if run_type == 'POSTGRESQL': data_dir = run_options.get( 'data_dir', os.path.join(please_cli.config.TMP_DIR, 'postgresql')) if not os.path.isdir(data_dir): click.echo( ' => Initialize database folder `{}` ... '.format(data_dir), nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke( please_cli.shell.cmd, project=project, command='initdb -D {} --auth=trust'.format(data_dir), nix_shell=nix_shell, ) please_cli.utils.check_result(result, output) schema = '' command = [ 'postgres', '-D', data_dir, '-h', host, '-p', port, ] elif run_type == 'FLASK': for env_name, env_value in run_options.get('envs', {}).items(): env_name = env_name.replace('-', '_').upper() os.environ[env_name] = env_value if not os.path.exists(ca_cert_file) or \ not os.path.exists(server_cert_file) or \ not os.path.exists(server_key_file): ctx.invoke( please_cli.create_certs.cmd, certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'), ) project_cache_dir = os.path.join(please_cli.config.TMP_DIR, 'cache', project_name) if not os.path.isdir(project_cache_dir): os.makedirs(project_cache_dir) os.environ['CACHE_TYPE'] = 'filesystem' os.environ['CACHE_DIR'] = project_cache_dir os.environ['APP_SETTINGS'] = os.path.join(please_cli.config.ROOT_DIR, 'src', project_name, 'settings.py') os.environ['APP_URL'] = '{}{}:{}'.format(schema, host, port) os.environ['CORS_ORIGINS'] = '*' command = [ 'gunicorn', project_name + '.flask:app', '--bind', '{}:{}'.format(host, port), '--ca-certs={}'.format(ca_cert_file), '--certfile={}'.format(server_cert_file), '--keyfile={}'.format(server_key_file), '--workers', '2', '--timeout', '3600', '--reload', '--reload-engine=poll', '--log-file', '-', ] elif run_type == 'SPHINX': schema = 'http://' command = [ 'HOST=' + host, 'PORT=' + port, 'python', 'run.py', ] elif run_type == 'ELM': if not os.path.exists(ca_cert_file) or \ not os.path.exists(server_cert_file) or \ not os.path.exists(server_key_file): ctx.invoke( please_cli.create_certs.cmd, certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'), ) os.environ['WEBPACK_RELEASE_VERSION'] = please_cli.config.VERSION os.environ['WEBPACK_RELEASE_CHANNEL'] = 'development' os.environ['SSL_CACERT'] = ca_cert_file os.environ['SSL_CERT'] = server_cert_file os.environ['SSL_KEY'] = server_key_file for env_name, env_value in run_options.get('envs', {}).items(): env_name = 'WEBPACK_' + env_name.replace('-', '_').upper() os.environ[env_name] = env_value # XXX: once we move please_cli.config.PROJECTS to nix we wont need this for require in project_config.get('requires', []): env_name = 'WEBPACK_{}_URL'.format( require.replace('-', '_').upper()) env_value = '{}://{}:{}'.format( please_cli.config.PROJECTS[require]['run_options'].get( 'schema', 'https'), please_cli.config.PROJECTS[require]['run_options'].get( 'host', host), please_cli.config.PROJECTS[require]['run_options']['port'], ) os.environ[env_name] = env_value command = [ 'webpack-dev-server', '--host', host, '--port', port, '--config', os.path.join(please_cli.config.ROOT_DIR, 'src', project_name, 'webpack.config.js'), ] click.echo(' => Running {} on {}{}:{} ...'.format(project, schema, host, port)) returncode, output, error = ctx.invoke( please_cli.shell.cmd, project=project, quiet=quiet, command=' '.join(command), nix_shell=nix_shell, taskcluster_secrets=taskcluster_secrets, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) sys.exit(returncode)
def update_organization(ctx, from_json, wait_for_state, max_wait_seconds, wait_interval_seconds, organization_id, default_ucm_subscription_id, if_match): if isinstance(organization_id, six.string_types) and len( organization_id.strip()) == 0: raise click.UsageError( 'Parameter --organization-id cannot be whitespace or empty string') kwargs = {} if if_match is not None: kwargs['if_match'] = if_match kwargs['opc_request_id'] = cli_util.use_or_generate_request_id( ctx.obj['request_id']) _details = {} _details['defaultUcmSubscriptionId'] = default_ucm_subscription_id client = cli_util.build_client('tenant_manager_control_plane', 'organization', ctx) result = client.update_organization(organization_id=organization_id, update_organization_details=_details, **kwargs) if wait_for_state: if hasattr(client, 'get_work_request') and callable( getattr(client, 'get_work_request')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs[ 'max_interval_seconds'] = wait_interval_seconds click.echo( 'Action completed. Waiting until the work request has entered state: {}' .format(wait_for_state), file=sys.stderr) result = oci.wait_until( client, client.get_work_request( result.headers['opc-work-request-id']), 'status', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo( 'Failed to wait until the work request entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo( 'Encountered error while waiting for work request to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo( 'Unable to wait for the work request to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
def update_appliance_export_job(ctx, from_json, force, wait_for_state, max_wait_seconds, wait_interval_seconds, appliance_export_job_id, bucket_name, prefix, range_start, range_end, display_name, lifecycle_state, lifecycle_state_details, manifest_file, manifest_md5, number_of_objects, total_size_in_bytes, first_object, last_object, next_object, customer_shipping_address, freeform_tags, defined_tags, if_match): if isinstance(appliance_export_job_id, six.string_types) and len(appliance_export_job_id.strip()) == 0: raise click.UsageError('Parameter --appliance-export-job-id cannot be whitespace or empty string') if not force: if customer_shipping_address or freeform_tags or defined_tags: if not click.confirm("WARNING: Updates to customer-shipping-address and freeform-tags and defined-tags will replace any existing values. Are you sure you want to continue?"): ctx.abort() kwargs = {} if if_match is not None: kwargs['if_match'] = if_match kwargs['opc_request_id'] = cli_util.use_or_generate_request_id(ctx.obj['request_id']) _details = {} if bucket_name is not None: _details['bucketName'] = bucket_name if prefix is not None: _details['prefix'] = prefix if range_start is not None: _details['rangeStart'] = range_start if range_end is not None: _details['rangeEnd'] = range_end if display_name is not None: _details['displayName'] = display_name if lifecycle_state is not None: _details['lifecycleState'] = lifecycle_state if lifecycle_state_details is not None: _details['lifecycleStateDetails'] = lifecycle_state_details if manifest_file is not None: _details['manifestFile'] = manifest_file if manifest_md5 is not None: _details['manifestMd5'] = manifest_md5 if number_of_objects is not None: _details['numberOfObjects'] = number_of_objects if total_size_in_bytes is not None: _details['totalSizeInBytes'] = total_size_in_bytes if first_object is not None: _details['firstObject'] = first_object if last_object is not None: _details['lastObject'] = last_object if next_object is not None: _details['nextObject'] = next_object if customer_shipping_address is not None: _details['customerShippingAddress'] = cli_util.parse_json_parameter("customer_shipping_address", customer_shipping_address) if freeform_tags is not None: _details['freeformTags'] = cli_util.parse_json_parameter("freeform_tags", freeform_tags) if defined_tags is not None: _details['definedTags'] = cli_util.parse_json_parameter("defined_tags", defined_tags) client = cli_util.build_client('dts', 'appliance_export_job', ctx) result = client.update_appliance_export_job( appliance_export_job_id=appliance_export_job_id, update_appliance_export_job_details=_details, **kwargs ) if wait_for_state: if hasattr(client, 'get_appliance_export_job') and callable(getattr(client, 'get_appliance_export_job')): try: wait_period_kwargs = {} if max_wait_seconds is not None: wait_period_kwargs['max_wait_seconds'] = max_wait_seconds if wait_interval_seconds is not None: wait_period_kwargs['max_interval_seconds'] = wait_interval_seconds click.echo('Action completed. Waiting until the resource has entered state: {}'.format(wait_for_state), file=sys.stderr) result = oci.wait_until(client, client.get_appliance_export_job(result.data.id), 'lifecycle_state', wait_for_state, **wait_period_kwargs) except oci.exceptions.MaximumWaitTimeExceeded as e: # If we fail, we should show an error, but we should still provide the information to the customer click.echo('Failed to wait until the resource entered the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) sys.exit(2) except Exception: click.echo('Encountered error while waiting for resource to enter the specified state. Outputting last known resource state', file=sys.stderr) cli_util.render_response(result, ctx) raise else: click.echo('Unable to wait for the resource to enter the specified state', file=sys.stderr) cli_util.render_response(result, ctx)
resumable=True) req = gcs_service.objects().insert( bucket=bucket, name=os.path.basename(archive.name), media_body=media, body={"cacheControl": "public,max-age=31536000"}) resp = None while resp is None: status, resp = req.next_chunk() if status: print "Uploaded %d%%." % int(status.progress() * 100) click.echo('...done!') except HttpError, amy: if amy.resp.status == 403: raise click.UsageError( 'You don\'t have permission to write to GCS bucket %s. Fix this or specify a different bucket to use.' % bucket) else: raise amy # Invoke the container builder API ccb_service = discovery.build( 'cloudbuild', 'v1', credentials=credentials, discoveryServiceUrl= "https://content-cloudbuild.googleapis.com/$discovery/rest?version=v1") req = ccb_service.projects().builds().create(projectId=project_id, body=cb_request_body)
def create_boot_volume_extended(ctx, **kwargs): if (kwargs['source_boot_volume_id'] and kwargs['boot_volume_backup_id']) or (kwargs['source_boot_volume_id'] and kwargs['source_volume_replica_id']) or \ (kwargs['boot_volume_backup_id'] and kwargs['source_volume_replica_id']): raise click.UsageError( 'You can only specify one of either --source-boot-volume-id, --boot-volume-backup-id or --source-volume-replica-id option' ) if not kwargs['source_boot_volume_id'] and not kwargs[ 'boot_volume_backup_id'] and not kwargs['source_volume_replica_id']: raise click.UsageError( 'An empty boot volume cannot be created. Please specify either --boot-volume-backup-id, --source-boot-volume-id or --source-volume-replica-id' ) if not kwargs['source_boot_volume_id'] and not kwargs[ 'source_volume_replica_id']: if not kwargs['availability_domain']: raise click.UsageError( 'An availability domain must be specified when restoring a boot volume from backup' ) client = cli_util.build_client('core', 'blockstorage', ctx) if kwargs['source_boot_volume_id']: source_boot_volume = client.get_boot_volume( boot_volume_id=kwargs['source_boot_volume_id']) if not kwargs['compartment_id']: kwargs['compartment_id'] = source_boot_volume.data.compartment_id if kwargs['boot_volume_backup_id']: source_backup = client.get_boot_volume_backup( boot_volume_backup_id=kwargs['boot_volume_backup_id']) if not kwargs['compartment_id']: kwargs['compartment_id'] = source_backup.data.compartment_id if kwargs['source_volume_replica_id']: source_volume_replica = client.get_boot_volume_replica( boot_volume_replica_id=kwargs['source_volume_replica_id']) kwargs[ 'availability_domain'] = source_volume_replica.data.availability_domain if not kwargs['compartment_id']: kwargs[ 'compartment_id'] = source_volume_replica.data.compartment_id if kwargs['source_boot_volume_id'] or kwargs[ 'boot_volume_backup_id'] or kwargs['source_volume_replica_id']: if kwargs['boot_volume_backup_id']: source_details = { 'type': 'bootVolumeBackup', 'id': kwargs['boot_volume_backup_id'] } elif kwargs['source_boot_volume_id']: source_details = { 'type': 'bootVolume', 'id': kwargs['source_boot_volume_id'] } else: source_details = { 'type': 'bootVolumeReplica', 'id': kwargs['source_volume_replica_id'] } kwargs['source_details'] = json.dumps(source_details) kwargs.pop('source_boot_volume_id', None) kwargs.pop('boot_volume_backup_id', None) kwargs.pop('source_volume_replica_id', None) json_skeleton_utils.remove_json_skeleton_params_from_dict(kwargs) ctx.invoke(blockstorage_cli.create_boot_volume, **kwargs)