def main(server, host, port, socket, processes, threads): """Starts the Supysonic web server""" if server is None: server = find_first_available_server() if server is None: raise ClickException( "Couldn't load any server, please install one of {}".format(_servers) ) else: try: server = get_server(server) except ImportError: raise ClickException( "Couldn't load {}, please install it first".format(server) ) if socket is not None: host = None port = None app = create_application() server( app, host=host, port=port, socket=socket, processes=processes, threads=threads ).run()
def bump_runner_extension_project(project_dir: str): click.secho(f'Bumping runner version on project {project_dir}...\n', fg='blue') latest_version = get_pypi_runner_version() docker_compose_file = os.path.join(project_dir, 'docker-compose.yml') if not os.path.isfile(docker_compose_file): raise ClickException( f'Mandatory `docker-compose.yml` file on directory `{project_dir}` is missing.' ) try: with open(docker_compose_file, 'r') as file_reader: data = yaml.load(file_reader, Loader=yaml.FullLoader) for service in data['services']: if 'image' in data['services'][service]: runner_image = data['services'][service]['image'] runner_version = runner_image.split(':')[-1] updated_image = runner_image.replace( runner_version, latest_version) data['services'][service]['image'] = updated_image with open(docker_compose_file, 'w') as file_writer: yaml.dump(data, file_writer, sort_keys=False) except yaml.YAMLError as error: raise ClickException( '`docker_compose.yml` file is not properly formatted. Please review it.\n' f'Error: {error}', ) click.secho( f'Runner version has been successfully updated to {latest_version}', fg='green')
def set_command( ctx: Context, json_path: str, value: str, type_: Optional[str], ): """Set a field.""" try: agent_config_manager = AgentConfigManager.load(ctx.cwd) current_value = None with contextlib.suppress(VariableDoesNotExist): current_value = agent_config_manager.get_variable(json_path) # type was not specified, tried to auto determine if type_ is None: # apply str as default type converted_value = convert_value_str_to_type(value, "str") if current_value is not None: # try to convert to original value's type with contextlib.suppress(Exception): converted_value = convert_value_str_to_type( value, type(current_value).__name__ ) else: # convert to type specified by user converted_value = convert_value_str_to_type(value, cast(str, type_)) agent_config_manager.set_variable(json_path, converted_value) agent_config_manager.dump_config() except ExtraPropertiesError as e: # pragma: nocover raise ClickException(f"Attribute `{e.args[0][0]}` is not allowed to change!") except (ValueError, AEAException) as e: raise ClickException(*e.args)
def register(ctx, project, paths, modules, json_paths, names, labels, force, watch, schedule): """Register one or more flows into a project. Flows with unchanged metadata will be skipped as registering again will only change the version number. """ # Since the old command was a subcommand of this, we have to do some # mucking to smoothly deprecate it. Can be removed with `prefect register # flow` is removed. if ctx.invoked_subcommand is not None: if any([project, paths, modules, names, labels, force]): raise ClickException("Got unexpected extra argument (%s)" % ctx.invoked_subcommand) return if project is None: raise ClickException("Missing required option '--project'") paths = expand_paths(paths) if watch: if any(parse_path(j).scheme != "file" for j in json_paths): raise ClickException("--watch is not supported for remote paths") json_paths = set(json_paths) ctx = multiprocessing.get_context("spawn") for change_paths_temp, change_mods in watch_for_changes( paths=paths, modules=modules): change_paths = [] change_json_paths = [] for p in change_paths_temp: if p in json_paths: change_json_paths.append(p) else: change_paths.append(p) proc = ctx.Process( target=register_internal, name="prefect-register", args=(project, ), kwargs=dict( paths=change_paths, modules=change_mods, json_paths=change_json_paths, names=names, labels=labels, force=force, in_watch=True, schedule=schedule, ), daemon=True, ) proc.start() proc.join() else: modules = list(modules or ()) register_internal(project, paths, modules, json_paths, names, labels, force, schedule)
def import_users(infile): """Import users.""" click.secho('Importing users from {file}'.format(file=infile.name)) data = json.load(infile) for user_data in data: try: email = user_data.get('email') # No email found in user's data, account cannot be created if not email: raise ClickException('Email not defined') user = datastore.find_user(email=email) # User already exists, skip account creation if user: raise ClickException( 'User with email {email} already exists'.format( email=email)) password = user_data.get( 'password', '$pbkdf2-sha512$25000$29ubk1KqFUJorTXmHAPAmA$ooj0RJyHyinmZw' '/.pNMXne8p70X/BDoX5Ypww24OIguSWEo3y.KT6hiwxwHS5OynZNkgnLvf' 'R3m1mNVfsHgfgA') del user_data['password'] if not user_data.get('role'): user_data['role'] = UserRecord.ROLE_USER if not datastore.find_role(user_data['role']): datastore.create_role(name=user_data['role']) datastore.commit() # Create account and activate it datastore.create_user(email=email, password=password, roles=[user_data['role']]) datastore.commit() user = datastore.find_user(email=email) confirm_user(user) datastore.commit() click.secho( 'User {email} with ID #{id} created successfully'.format( email=email, id=user.id), fg='green') # Create user resource user = UserRecord.create(user_data, dbcommit=True) user.reindex() except Exception as error: click.secho('User {user} could not be imported: {error}'.format( user=user_data, error=str(error)), fg='red') click.secho('Finished', fg='green')
def import_users(infile): """Import users.""" click.secho('Importing users from {file}'.format(file=infile)) data = json.load(infile) for user_data in data: try: email = user_data.get('email') # No email found in user's data, account cannot be created if not email: raise ClickException('Email not defined') user = datastore.find_user(email=email) # User already exists, skip account creation if user: raise ClickException( 'User with email {email} already exists'.format( email=email)) password = user_data.get('password', '123456') password = hash_password(password) del user_data['password'] if not user_data.get('roles'): user_data['roles'] = [UserRecord.ROLE_USER] roles = user_data.get('roles', []).copy() for role in roles: if not datastore.find_role(role): datastore.create_role(name=role) datastore.commit() # Create account and activate it datastore.create_user(email=email, password=password, roles=roles) datastore.commit() user = datastore.find_user(email=email) confirm_user(user) datastore.commit() click.secho( 'User {email} with ID #{id} created successfully'.format( email=email, id=user.id), fg='green') # Create user resource user = UserRecord.create(user_data, dbcommit=True) user.reindex() except Exception as error: click.secho( 'User {user} could not be imported: {error}'.format( user=user_data, error=str(error)), fg='red') click.secho('Finished', fg='green')
def get_active_tmux_session() -> str: if "TMUX" not in os.environ: raise ClickException("When run outside of tmux use the -s flag.") process = subprocess.Popen(["tmux", "display-message", "-p", "#S"], stdout=subprocess.PIPE) output = process.communicate()[0] if process.returncode != 0: raise ClickException("Failed to get tmux session id.") return output.decode("utf-8").strip()
def register(project, paths, modules, json_paths, names, labels, force, watch, schedule): """Register one or more flows into a project. Flows with unchanged metadata will be skipped as registering again will only change the version number. """ if project is None: raise ClickException("Missing required option '--project'") paths = expand_paths(paths) if watch: if any(parse_path(j).scheme != "file" for j in json_paths): raise ClickException("--watch is not supported for remote paths") json_paths = set(json_paths) ctx = multiprocessing.get_context("spawn") for change_paths_temp, change_mods in watch_for_changes( paths=paths, modules=modules): change_paths = [] change_json_paths = [] for p in change_paths_temp: if p in json_paths: change_json_paths.append(p) else: change_paths.append(p) proc = ctx.Process( target=register_internal, name="prefect-register", args=(project, ), kwargs=dict( paths=change_paths, modules=change_mods, json_paths=change_json_paths, names=names, labels=labels, force=force, in_watch=True, schedule=schedule, ), daemon=True, ) proc.start() proc.join() else: modules = list(modules or ()) register_internal(project, paths, modules, json_paths, names, labels, force, schedule)
def validate(ctx): """ Validate our configuration """ if not config.contentful: raise ClickException( "Contentful is not configured, please specify the CONTENTFUL_SPACE_ID and " "CONTENTFUL_ACCESS_TOKEN environment variables.") space = config.contentful.space() codes = [x.code for x in space.locales] primary = [x.code for x in space.locales if x.default] primary = primary[0] if primary else (codes[0] if codes else 'N/A') click.echo("Current config:") click.echo(" CONTENTFUL_LANGUAGES=%s" % ','.join(sorted(config.LANGUAGES))) click.echo(" CONTENTFUL_DEFAULT_LANGUAGE=%s" % config.DEFAULT_LANGUAGE) click.echo("Discovered:") click.echo(" CONTENTFUL_LANGUAGES=%s" % ','.join(sorted(codes))) click.echo(" CONTENTFUL_DEFAULT_LANG=%s" % primary) if not set(config.LANGUAGES) == set(codes): click.echo("Please update CONTENTFUL_LANGUAGES to match") ctx.exit(1) if not config.DEFAULT_LANGUAGE == primary: click.echo("Please update CONTENTFUL_DEFAULT_LANGUAGE to match") ctx.exit(1) click.echo("Configuration matches") ctx.exit(0)
def full_import(verbose, force): """ Imports the entire space if the database is empty --- Use the --force toggle to force this regardless of data already existing in the database """ doc = {"query": {"match_all": {}}} if not config.contentful: raise ClickException( "Contentful is not configured, please specify the CONTENTFUL_SPACE_ID and " "CONTENTFUL_ACCESS_TOKEN environment variables.") if not force: try: results = config.elastic.search( index=config.content_type_index(), body=doc) if len(results["hits"]["hits"]) > 0: click.echo( f"Data already exists in database, skipping import. Use --force to force import on an existing database." ) return except NotFoundError: # This happens when the index does not exist, at which point we have an empty database. click.echo( f"Index {config.content_type_index()} does not exist. Assume this is an empty database." ) pass click.echo(f"Importing all content types.") _update(verbose=verbose) click.echo(f"Importing all content. This might take a while...") _import_all_documents(verbose=verbose)
def get_command(self, ctx, name): """Imports the command if available in commmands list and provides with most similar commands if the command is not present in the list. Args: name(str): The name of the command. Returns: click.core.Command: It is a new command and uses the decorated function as callback.For reference visit https://click.palletsprojects.com/en/7.x/api/#decorators . """ try: if sys.version_info[0] == 2: name = name.encode("ascii", "replace") mod = __import__("popper.commands.cmd_" + name, None, None, ["cli"]) except ImportError as e: commands = self.list_commands(ctx) most_similar_commands = ", ".join( difflib.get_close_matches(name, commands, 3, 0.3)) message = "" if len(most_similar_commands) != 0: message = "\n\nThe most similar commands are: " + most_similar_commands raise ClickException("Command '" + name + "' doesn't exist.\n" "Type 'popper --help' for more.\n" + message + "\n" + str(e)) return mod.cli
def gc(ctx, keep): """Garbage collector, cleans up your local run directory""" directory = wandb.wandb_dir() if not os.path.exists(directory): raise ClickException('No wandb directory found at %s' % directory) paths = glob.glob(directory + "/*run*") dates = [ datetime.datetime.strptime(p.split("-")[1], '%Y%m%d_%H%M%S') for p in paths ] since = datetime.datetime.utcnow() - datetime.timedelta(hours=keep) bad_paths = [paths[i] for i, d, in enumerate(dates) if d < since] if len(bad_paths) > 0: click.echo("Found {} runs, {} are older than {} hours".format( len(paths), len(bad_paths), keep)) click.confirm(click.style("Are you sure you want to remove %i runs?" % len(bad_paths), bold=True), abort=True) for path in bad_paths: shutil.rmtree(path) click.echo(click.style("Success!", fg="green")) else: click.echo( click.style("No runs older than %i hours found" % keep, fg="red"))
def pull(run, project, entity): project, run = api.parse_slug(run, project=project) urls = api.download_urls(project, run=run, entity=entity) if len(urls) == 0: raise ClickException("Run has no files") click.echo("Downloading: {project}/{run}".format(project=click.style( project, bold=True), run=run)) for name in urls: if api.file_current(name, urls[name]['md5']): click.echo("File %s is up to date" % name) else: length, response = api.download_file(urls[name]['url']) # TODO: I had to add this because some versions in CI broke click.progressbar sys.stdout.write("File %s\r" % name) with click.progressbar(length=length, label='File %s' % name, fill_char=click.style('&', fg='green')) as bar: with open(name, "wb") as f: for data in response.iter_content(chunk_size=4096): f.write(data) bar.update(len(data))
def prompt_for_project(ctx, entity): """Ask the user for a project, creating one if necessary.""" result = ctx.invoke(projects, entity=entity, display=False) try: if len(result) == 0: project = click.prompt("Enter a name for your first project") #description = editor() project = api.upsert_project(project, entity=entity)["name"] else: project_names = [project["name"] for project in result] question = { 'type': 'list', 'name': 'project_name', 'message': "Which project should we use?", 'choices': project_names + ["Create New"] } result = whaaaaat.prompt([question]) if result: project = result['project_name'] else: project = "Create New" # TODO: check with the server if the project exists if project == "Create New": project = click.prompt("Enter a name for your new project", value_proc=api.format_project) #description = editor() project = api.upsert_project(project, entity=entity)["name"] except wandb.apis.CommError as e: raise ClickException(str(e)) return project
def import_organisations(file): """Import organisations from JSON file.""" click.secho('Importing organisations from {file}'.format(file=file.name)) indexer = OrganisationIndexer() for record in json.load(file): try: # Check existence in DB db_record = OrganisationRecord.get_record_by_pid(record['code']) if db_record: raise ClickException('Record already exists in DB') # Register record to DB db_record = OrganisationRecord.create(record) db.session.commit() indexer.index(db_record) except Exception as error: click.secho( 'Organisation {org} could not be imported: {error}'.format( org=record, error=str(error)), fg='red') click.secho('Finished', fg='green')
def pull(run, project, entity): api = InternalApi() project, run = api.parse_slug(run, project=project) urls = api.download_urls(project, run=run, entity=entity) if len(urls) == 0: raise ClickException("Run has no files") click.echo( "Downloading: {project}/{run}".format( project=click.style(project, bold=True), run=run ) ) for name in urls: if api.file_current(name, urls[name]["md5"]): click.echo("File %s is up to date" % name) else: length, response = api.download_file(urls[name]["url"]) # TODO: I had to add this because some versions in CI broke click.progressbar sys.stdout.write("File %s\r" % name) dirname = os.path.dirname(name) if dirname != "": wandb.util.mkdir_exists_ok(dirname) with click.progressbar( length=length, label="File %s" % name, fill_char=click.style("&", fg="green"), ) as bar: with open(name, "wb") as f: for data in response.iter_content(chunk_size=4096): f.write(data) bar.update(len(data))
def get(path, root, type): public_api = PublicApi() entity, project, artifact_name = public_api._parse_artifact_path(path) if project is None: project = click.prompt("Enter the name of the project you want to use") try: artifact_parts = artifact_name.split(":") if len(artifact_parts) > 1: version = artifact_parts[1] artifact_name = artifact_parts[0] else: version = "latest" full_path = "{entity}/{project}/{artifact}:{version}".format( entity=entity, project=project, artifact=artifact_name, version=version ) wandb.termlog( "Downloading {type} artifact {full_path}".format( type=type or "dataset", full_path=full_path ) ) artifact = public_api.artifact(full_path, type=type) path = artifact.download(root=root) wandb.termlog("Artifact downloaded to %s" % path) except ValueError: raise ClickException("Unable to download artifact")
def login(key, host, cloud, relogin, anonymously, no_offline=False): # TODO: handle no_offline anon_mode = "must" if anonymously else "never" wandb.setup(settings=wandb.Settings( _cli_only_mode=True, anonymous=anon_mode, base_url=host)) api = _get_cling_api() if host == "https://api.wandb.ai" or (host is None and cloud): api.clear_setting("base_url", globally=True, persist=True) # To avoid writing an empty local settings file, we only clear if it exists if os.path.exists(Settings._local_path()): api.clear_setting("base_url", persist=True) elif host: if not host.startswith("http"): raise ClickException("host must start with http(s)://") api.set_setting("base_url", host.strip("/"), globally=True, persist=True) key = key[0] if len(key) > 0 else None wandb.login(relogin=relogin, key=key, anonymous=anon_mode, host=host, force=True)
def get_cluster_connection(path: str = CONFIG_PATH) -> Cluster: current_cluster = read_current_cluster(path) if current_cluster is None: raise ClickException( "Can't establish connection to Hydrosphere cluster: cluster config is missing. Use `hs cluster` commands." ) return Cluster(http_address=current_cluster.cluster.server)
def tmux_broadcast(session: Optional[str], command: str): if session is None: session = get_active_tmux_session() window_ids = get_tmux_window_ids(session) for window_id in window_ids: tmux_commands = [ [ "tmux", "setw", "-t", f"{session}:{window_id}", "synchronize-panes" ], ["tmux", "send-keys", "-lt", f"{session}:{window_id}", command], ["tmux", "send-keys", "-t", f"{session}:{window_id}", "Enter"], [ "tmux", "setw", "-t", f"{session}:{window_id}", "synchronize-panes", "off", ], ] for tmux_command in tmux_commands: process = subprocess.Popen(tmux_command, stdout=subprocess.PIPE) process.communicate() if process.returncode != 0: raise ClickException("Failed to get tmux window list.")
def validate(source: str, version: str, output_csv: bool, schemas: Path): """ Validate a transit data file against a specified schema. """ if schemas: _load_schemas(schemas) try: validator = Validators.get_validator(version) except TransidateException as exc: console.rule(HEADER + "Error") raise ClickException(str(exc)) dataset_path = Path(source) dataset = DataSet(dataset_path) result = _validate(validator, dataset) if result.status == ValidationResult.OK: console.rule(HEADER + "Results") console.print("No issues found.") else: console.rule(HEADER + f"Results: {len(result.violations)} Issues found") ConsoleOutput(dataset=dataset, result=result).output() if output_csv: CSVOutput(dataset=dataset, result=result).output()
def delete_document(content_type, docid): """ Removes a single document from the index. --- Please note that the storage/removal of documents depends on the CONTENTFUL_ACCESS_TOKEN's access. It is _imperative_ that you do not use this method with a PREVIEW token, when ENABLE_UNPUBLISHED is False! """ if not config.contentful: raise ClickException( "Contentful is not configured, please specify the CONTENTFUL_SPACE_ID and " "CONTENTFUL_ACCESS_TOKEN environment variables.") data = { 'sys': { 'contentType': { 'sys': { 'id': content_type } }, 'id': docid, }, } obj = Entry(data) if obj.valid_for_space(): obj.delete() else: click.echo("Entry is not valid for space") return
def import_institutions(): """Import institutions from JSON file.""" institution_file = './data/institutions.json' click.secho( 'Importing institution from {file}'.format(file=institution_file)) indexer = RecordIndexer() with open(institution_file) as json_file: records = json.load(json_file) for record in records: try: # Check existence in DB db_record = InstitutionRecord.get_record_by_pid(record['pid']) if db_record: raise ClickException('Record already exists in DB') # Register record to DB db_record = InstitutionRecord.create(record) db.session.commit() indexer.index(db_record) except Exception as error: click.secho( 'Institution {institution} could not be imported: {error}'. format(institution=record, error=str(error)), fg='red') click.secho('Finished', fg='green')
def publish(gh_token: str) -> None: """Publish release on GitHub.""" # Make sure we're in a git repo. get_repo_and_check_clean_checkout() current_version = get_package_version() tag_name = f"v{current_version}" if not click.confirm(f"Publish release {tag_name} on GitHub?", default=True): return # Publish the draft release gh = Github(gh_token) gh_repo = gh.get_repo("matrix-org/synapse") for release in gh_repo.get_releases(): if release.title == tag_name: break else: raise ClickException(f"Failed to find GitHub release for {tag_name}") assert release.title == tag_name if not release.draft: click.echo("Release already published.") return release = release.update_release( name=release.title, message=release.body, tag_name=release.tag_name, prerelease=release.prerelease, draft=False, )
def folder_scan(config, folder, force, mode): """Run a scan on specified folders. FOLDER is the name of the folder to scan. Multiple can be specified. If ommitted, all folders are scanned. """ daemon = DaemonClient(config.DAEMON["socket"]) # quick and dirty shorthand calls scan_bg = lambda: daemon.scan(folder, force) scan_fg = lambda: _folder_scan_foreground(config, daemon, folder, force) auto = not mode if auto: try: scan_bg() except DaemonUnavailableError: click.echo( "Couldn't connect to the daemon, scanning in foreground", err=True) scan_fg() elif mode == "background": try: scan_bg() except DaemonUnavailableError as e: raise ClickException( "Couldn't connect to the daemon, please use the '--foreground' option", ) from e elif mode == "foreground": scan_fg()
def _add(cfg, env_name): try: pkg_name = default.find_package_name() except ValueError as e: raise ClickException( 'AWS Lambda is only supported in packaged projects. ' 'See the documentation for an example.') from e with Commander(workspace=env_name, templates_path=('soopervisor', 'assets')) as e: e.copy_template('aws-lambda/README.md') e.copy_template('aws-lambda/Dockerfile') e.copy_template('aws-lambda/test_aws_lambda.py', package_name=pkg_name) e.copy_template('aws-lambda/app.py', package_name=pkg_name) e.copy_template('aws-lambda/template.yaml', package_name=pkg_name) e.success('Done.') e.print( 'Next steps:\n1. Add an input example to ' f'{env_name}/test_aws_lambda.py\n' f'2. Add the input parsing logic to {env_name}/app.py\n' f'3. Submit to AWS Lambda with: soopervisor export {env_name}') # TODO: use e.warn_on_exit for name in ['docker', 'aws', 'sam']: warn_if_not_installed(name)
def sync(ctx, path, id, project, entity, ignore): if api.api_key is None: ctx.invoke(login) if ignore: globs = ignore.split(",") else: globs = None path = path[0] if len(path) > 0 else os.getcwd() if os.path.isfile(path): raise ClickException("path must be a directory") wandb_dir = os.path.join(path, "wandb") run_paths = glob.glob(os.path.join(wandb_dir, "*run-*")) if len(run_paths) == 0: run_paths = glob.glob(os.path.join(path, "*run-*")) if len(run_paths) > 0: for run_path in run_paths: wandb_run.Run.from_directory(run_path, run_id=run_path.split("-")[-1], project=project, entity=entity, ignore_globs=globs) else: wandb_run.Run.from_directory(path, run_id=id, project=project, entity=entity, ignore_globs=globs)
def run_post_install(filename: Path, post_install: str): if post_install is not None: os.chdir(filename.parent) try: subprocess.run(post_install, shell=True, check=True) except subprocess.CalledProcessError as e: raise ClickException(str(e)) from e
def test_decrypt_folder_error(self): self.cwa_m.side_effect = ClickException("error") with pytest.raises(ClickException): decrypt_folder("filepath") self.cwa_m.assert_called_once() self.decr_file_m.assert_not_called()
def register(ctx, project, paths, modules, names, labels, force, watch): """Register one or more flows into a project.""" # Since the old command was a subcommand of this, we have to do some # mucking to smoothly deprecate it. Can be removed with `prefect register # flow` is removed. if ctx.invoked_subcommand is not None: if any([project, paths, modules, names, labels, force]): raise ClickException("Got unexpected extra argument (%s)" % ctx.invoked_subcommand) return if project is None: raise ClickException("Missing required option '--project'") try: if watch: ctx = multiprocessing.get_context("spawn") for change_paths, change_mods in watch_for_changes( paths=paths, modules=modules): proc = ctx.Process( target=register_internal, name="prefect-register", args=(project, ), kwargs=dict( paths=change_paths, modules=change_mods, names=names, labels=labels, force=force, in_watch=True, ), daemon=True, ) proc.start() proc.join() else: paths = expand_paths(list(paths or ())) modules = list(modules or ()) register_internal(project, paths, modules, names, labels, force) except TerminalError as exc: msg = str(exc) if msg: click.secho(msg, fg="red") sys.exit(1)
def __init__(self, message): ClickException.__init__(self, message) self.message = message