def output( print_json: bool = Option(False, "--json", help=_json_help), organization: str = Option("", "-o", "--organization", help=_organization_help), environment: str = Option("", "-e", "--environment", help=_environment_help), graph: Path = Option("", "-g", "--graph-path", help=_graph_help), store_name: str = Argument(..., exists=True, help=_store_name_help), ): """List data sent to an output port of a from the most recent run of a node""" ids = IdLookup( environment_name=environment, organization_name=organization, explicit_graph_path=graph, ) with abort_on_error("Could not get node data"): data = list( paginated_output_data(ids.environment_id, ids.graph_id, store_name) ) _print_objects(data, print_json)
def branch( repository: Path = Argument(..., help="The path to the bare repository."), local: bool = True, remote: bool = False, ) -> None: """Get the branches of a repository.""" from pygit2 import Repository # pylint: disable=import-outside-toplevel from typer import echo # pylint: disable=import-outside-toplevel repo = Repository(repository) branches: Iterable[str] if local and remote: branches = repo.branches elif local: branches = repo.branches.local elif remote: branches = repo.branches.remote for branch_name in branches: echo(branch_name)
def raster_tile_cache( dataset: str = Option(..., help="Dataset name."), version: str = Option(..., help="Version number."), zoom_level: int = Option(..., help="Zoom level."), implementation: str = Option( ..., help="Implementation name/pixel meaning."), target_bucket: str = Option(..., help="Target bucket."), skip_empty_tiles: bool = Option( False, "--skip_empty_tiles", help="Do not write empty tiles to tile cache."), bit_depth: int = Option(8, help="Number of bits per channel to use."), tile_set_prefix: str = Argument(..., help="Tile prefix."), ): LOGGER.info(f"Raster tile set asset prefix: {tile_set_prefix}") tiles: List[Tuple[str, str]] = get_input_tiles(tile_set_prefix) # If there are no files, what can we do? Just exit I guess! if not tiles: LOGGER.info("No input files! I guess we're good then?") return sub_processes = max(math.floor(NUM_PROCESSES / len(tiles)), 1) LOGGER.info(f"Using {sub_processes} sub-processes per process") args = (( tile, dataset, version, target_bucket, implementation, zoom_level, skip_empty_tiles, sub_processes, bit_depth, ) for tile in tiles) # Cannot use normal pool here, since we run sub-processes # https://stackoverflow.com/a/61470465/1410317 with ProcessPoolExecutor(max_workers=NUM_PROCESSES) as executor: for tile in executor.map(create_tiles, args): LOGGER.info(f"Finished processing tile {tile}")
def doi_process( ids: List[str] = Argument(..., help="Valid DOI/arXivID(s) or file (*.bib, *.txt)"), dir: Path = Option(getcwd, '--dir', '-d', help="Directory to download"), scihub: str = Option(config.__scihub__, '--scihub', '-s', help="Valid Sci-Hub URL") ): global operator try: assert not scihub.startswith( "http"), 'Argument Error: Invalid URL, example: sci-hub.tf' scihub = "https://" + scihub assert path.exists(dir), 'Argument Error: Invalid path.' if ids[0].lower().endswith('.bib'): assert path.exists(ids[0]), 'Argument Error: Invalid file path.' ids_list = operator.parseBibTex(ids[0]) elif ids[0].lower().endswith('.txt'): assert path.exists(ids[0]), 'Argument Error: Invalid file path.' ids_list = operator.parseTxt(ids[0]) else: ids_list = ids except AssertionError as e: echo(e.args[0], err=True) raise Exit() if not ids_list: echo("There is no valid DOI.", err=True) raise Exit() articles = list() for d in ids_list: if d.startswith('arXiv'): articles.append({ "article_url": urljoin('https://arxiv.org/abs', d.split(':')[1]), "file_name": f"{d.replace(':', '-')}.pdf", "warning_str": d}) else: articles.append({ "article_url": urljoin(scihub, d), "file_name": f"{d.replace('/', '-')}.pdf", "warning_str": d}) task = "ID" missing, log_file = operator.download(task, articles, dir) echo(f" {' '*len(task)} | {missing} missing: {log_file}")
def webhook( explicit_graph: Path = Option(None, "--graph", "-g", exists=True, help=_graph_help), name: str = Argument(..., help=_webhook_name_help), ): """Add a new webhook node to a graph""" ids = IdLookup(explicit_graph_path=explicit_graph) with abort_on_error("Adding webhook failed"): editor = GraphConfigEditor(ids.graph_file_path) editor.add_webhook(name, id=random_node_id()) editor.write() sprint(f"\n[success]Created webhook [b]{name}") sprint( f"\n[info]Once you've deployed the graph, use " f"[code]patterns list webhooks[/code] to get the url of the webhook")
def generate_keys( domain: str = Argument( ..., help="Domain to generate DKIM keys for"), # noqa: B008 ) -> None: client = from_env() build_kolombo_image(client, "dkim-gen") step(f"Generating DKIM keys for domain: {domain}") client.containers.run( "kolombo-dkim-gen", domain, stderr=True, auto_remove=True, volumes=["/etc/kolombo/dkim_keys:/etc/opendkim/keys"], ) dkim_txt = read_dkim_txt_record(domain) info( f"[b]TXT[/] record for [b u]mail._domainkey.{domain}[/] is: {dkim_txt}" )
def delete( graph_id: str = Option(""), force: bool = Option(False, "-f", "--force", help=_force_help), graph: Path = Argument(None, exists=True, help=_graph_help), ): """Delete a graph from the Patterns studio. This will not delete any files locally. """ ids = IdLookup( explicit_graph_path=graph, explicit_graph_id=graph_id, ) with abort_on_error("Deleting graph failed"): if not force: Confirm.ask(f"Delete graph {ids.graph_name}?") delete_graph(ids.graph_id) sprint(f"[success]Graph deleted from Patterns studio.")
def in_( when: str = Argument(None, help="Set a specific time to clock in"), out: str = Option(None, help="Set time to clock out"), m: str = Option(None, help="Journal Message to add to record"), ): """Clock into a job, or add a job day""" if when is not None and len(when.split("-")) > 3: when, out = parse_date_time_junction(when) else: if out is not None: out = parse_date_and_time(out) if when is not None: when = parse_date_and_time(when) if when is not None and out is not None: print( f"Creating entry for {when:%Y-%m-%d}: {when:%H:%M:%S} to {out:%H:%M:%S}" ) c = Clok( time_in=when, time_out=out, date_key=get_date_key(when), month_key=get_month(when), week_key=get_week(when), ) c.update_span() c.save() if m is not None: c.add_journal(m) elif when is not None and out is None: Clok.clock_in_when(when, verbose=True) if m is not None: Clok.get_last_record().add_journal(m) State.set_clok(Clok.get_last_record()) else: Clok.clock_in(verbose=True) if m is not None: Clok.get_last_record().add_journal(m) State.set_clok(Clok.get_last_record())
def main( db_path: Path = Argument(Path("pypms.sqlite"), help="sensor messages DB"), samples: int = Option(4, "--samples", "-n"), interval: int = Option(20, "--interval", "-i"), ): """ Read raw messages from 2 different sensors (PMSx003 on /dev/ttyUSB0 and MCU680 on /dev/ttyUSB1) and store them on a sqlite DB. After reading the sensor, decode all messages on DB and print them. """ # get DB context manager message_db = pypms_db(db_path) reader = dict( pms=SensorReader("PMSx003", "/dev/ttyUSB0", interval, samples), bme=SensorReader("MCU680", "/dev/ttyUSB1", interval, samples), ) # read from each sensor and write to DB with message_db() as db, reader["pms"] as pms, reader["bme"] as bme: # read one obs from each sensor at the time with progressbar(zip(pms(raw=True), bme(raw=True)), length=samples, label="reading sensors") as progress: for pms_obs, env_obs in progress: write_message(db, pms.sensor, pms_obs) write_message(db, bme.sensor, env_obs) # read and decode all messages on the DB with message_db() as db: # extract obs from one sensor at the time for sensor in [r.sensor for r in reader.values()]: print(sensor) for obs in read_obs(db, sensor): print(obs)
def cheapest( search_term: str = Argument(..., help="Search term for product."), n: int = Option(default=5, help="Max. number of cheapest findings to show."), username: str = Option(None, help="Username of Picnic account."), password: str = Option(None, help="Password of Picnic account."), country_code: str = Option(default="DE", help="Country code of shop."), ): # connect to picnic api if (not username) or (not password): print("loading credentials from config file...") username, password, country_code = credentials_from_file( path=CREDENTIALS_PATH) picnic = PicnicAPI(username=username, password=password, country_code=country_code) # search items, filter cheapest items = search_items(picnic_api=picnic, term=search_term) cheapest_items = filter_cheapest_items(items=items, max_n=n) # format results for item in cheapest_items: print(f"* {item.name}: {item.unit_quantity_sub}")
def heat( top: float = Argument( ..., help='The heat diffusion coefficient for the top bars.'), fv: bool = FV, grid: int = GRID, nt: int = NT, rect: bool = RECT, ): problem = InstationaryProblem( StationaryProblem( domain=RectDomain(top='dirichlet', bottom='neumann'), diffusion=LincombFunction( [ ConstantFunction(1., dim_domain=2), ExpressionFunction( '(x[..., 0] > 0.45) * (x[..., 0] < 0.55) * (x[..., 1] < 0.7) * 1.', dim_domain=2), ExpressionFunction( '(x[..., 0] > 0.35) * (x[..., 0] < 0.40) * (x[..., 1] > 0.3) * 1. + ' '(x[..., 0] > 0.60) * (x[..., 0] < 0.65) * (x[..., 1] > 0.3) * 1.', dim_domain=2) ], [ 1., 100. - 1., ExpressionParameterFunctional('top - 1.', {'top': 1}) ]), rhs=ConstantFunction(value=0., dim_domain=2), dirichlet_data=ConstantFunction(value=0., dim_domain=2), neumann_data=ExpressionFunction( '(x[..., 0] > 0.45) * (x[..., 0] < 0.55) * -1000.', dim_domain= 2), ), T=1., initial_data=ExpressionFunction( '(x[..., 0] > 0.45) * (x[..., 0] < 0.55) * (x[..., 1] < 0.7) * 10.', dim_domain=2)) mu = {'top': top} solve(problem, mu, fv, rect, grid, nt)
async def add_user( email: str = Argument(..., help="Email for new user"), # noqa: B008 ) -> None: from kolombo.models import Domain, User if "@" not in email: error(f"Email '{email}' does not contain '@'!") exit(1) domain = email.split("@", maxsplit=1)[1].strip() if not domain: error("Domain part MUST NOT be empty string!") exit(1) elif not await Domain.objects.filter(active=True, actual=domain).exists(): error(f"Domain '{domain}' is not added (or inactive)!") warning( f"You can add it via [code]kolombo domain add {domain} mx.{domain}[/code]" ) exit(1) elif await User.objects.filter(email=email).exists(): error(f"User with email '{email}' already exists!") exit(1) started(f"Adding [code]{email}[/] user") password = prompt(f"{email} password", hide_input=True, confirmation_prompt=True) step("Saving to database") await _save_user(email, password, domain) step("Updating virtual files (addresses and mailbox map)") active_users = await User.all_active() update_virtual_files(active_users) warning("Run command [code]kolombo run[/] to reload Kolombo") finished(f"User '{email}' added!")
def add( name: str = Argument(..., help="Name of the peer."), config_path: Path = common.OPTION_CONFIG_PATH, port: Optional[int] = common.OPTION_PORT, ipv4_address: Optional[str] = common.OPTION_IPV4_ADDRESS, ipv6_address: Optional[str] = common.OPTION_IPV6_ADDRESS, site: Optional[str] = None, ): """ Add a new peer. """ config = MainConfig.load(config_path) try: config.add_peer( name, IPv4Address(ipv4_address) if ipv4_address else None, IPv6Address(ipv6_address) if ipv6_address else None, port, ) except DuplicatePeerError as e: echo(str(e), err=True) config.save(config_path)
def install(ext_path: str = Argument( Path(), help="The path of the extension")) -> None: """Build and install an extension""" if not GLOBAL_EXTENSIONS_DIR.exists(): os.mkdir(GLOBAL_EXTENSIONS_DIR) extension_path = Path(ext_path).resolve() assert extension_path.joinpath("package.json").exists() _build_extension(extension_path, True, False) module, metadata = _get_extensions_metadata(extension_path) src = Path(extension_path).joinpath(module.__name__, metadata[0]["src"]) dest = GLOBAL_EXTENSIONS_DIR.joinpath(metadata[0]["dest"]) clean_dir(dest) shutil.copytree(src, dest, symlinks=True) print(f""" Extension installed: Path: {dest} """)
def main( network_type: NetworkType = Argument(..., help="type of the VAE network"), bottleneck_dim: int = Option(16, "--bottleneck_dim", "-n", help="size of the VAE bottleneck"), lr: float = Option(0.001, "--lr", "-r", help="learning rate for training"), batch_size: int = Option(..., "--batch_size", "-b", help="batch size for training"), epochs: int = Option(..., "--epochs", "-e", help="epochs to train"), device: str = Option("cpu", "--device", "-d", help='device to train on, e.g. "cuda:0"'), logdir: str = Option( "./results", "--logdir", "-l", help="directory to log the models and event file to", ), ): """Run the training for a VAE. """ mnist_data = dataset.MyMNIST() if network_type == NetworkType.mlp: net = model.MLPVAE((1, 32, 32), bottleneck_dim) else: net = model.CNNVAE((1, 32, 32), bottleneck_dim) optim = torch.optim.Adam(net.parameters(), lr) vae_trainer = trainer.Trainer(net, mnist_data, optim, batch_size, device, logdir) vae_trainer.train(epochs)
def assemble( filepth: Path = Argument( ..., exists=True, file_okay=True, dir_okay=True, readable=True, resolve_path=True, ), out: Path = Option(None), ): if filepth.suffix != ".asm": typer.echo("The file name must end with `.asm`") raise typer.Exit(code=1) if out is None: out = filepth.parent.joinpath(f"{filepth.stem}.hack") try: parser = Parser(filepth.read_text()) except ValueError as err: typer.echo(err) raise typer.Exit(1) assembler = Assembler(parser) try: assembly = assembler.assemble() except (InvalidCommandException, InvalidMnemonicError, AddressOutOfRange) as err: typer.echo(err) raise typer.Exit(code=1) typer.echo(f"Writing to {out}") with out.open("w") as f: f.writelines([x + "\n" for x in assembly]) typer.echo("Done")
def start(service: Service = Argument(..., help="Which service to start")) -> None: """ Start one of the memezer services. """ if service == Service.wsgi: import sys import time from signal import SIGINT, SIGTERM, signal from subprocess import Popen run_args = [ "gunicorn", "-k", "uvicorn.workers.UvicornWorker", "--config", "gunicorn.conf.py", "memezer.app:wsgi", ] gunicorn_master_proc = Popen(run_args) def kill_proc(signum: Any, frame: Any) -> None: gunicorn_master_proc.terminate() gunicorn_master_proc.wait() sys.exit(0) signal(SIGINT, kill_proc) signal(SIGTERM, kill_proc) while True: time.sleep(1) if service == Service.worker: from .queue import queue with queue.open(): queue.run_worker() # type: ignore
def install_component(component_file: str = Argument( ..., help="The path to the component file containing the data")): """Function to install a component: - Takes as input a YAML file - Parses it as a dictionary through the Pydantic model TunableComponentsModel - Send it as a POST request to /components """ api_client = Client( base_url=f"http://{api_settings.api_host}:{api_settings.api_port}", proxies={}, ) component = TunableComponentsModel.from_yaml(component_file) logger.debug(f"Sending component data {component.dict()}" "to endpoint" f"http: // {api_settings.api_host}: {api_settings.api_port}" f"{api_settings.component_endpoint}") request = api_client.post(api_settings.component_endpoint, json=component.dict()) logger.info("Successfully registered components.") if not 200 <= request.status_code < 400: raise Exception("Could not create component with status code" f"{request.status_code}")
def main( db_path: Path = Argument(Path("pypms.sqlite"), help="sensor measurements DB"), samples: int = Option(4, "--samples", "-n"), interval: int = Option(20, "--interval", "-i"), ): """ Read measurements from 2 different sensors (PMSx003 on /dev/ttyUSB0 and MCU680 on /dev/ttyUSB1) and store them on a sqlite DB as a "tall table" with a "wide table" view for each sensor. After reading the sensors, get all measurements from the DB amd print them by sensor. """ # get DB context manager measurements_db = pypms_db(db_path) reader = dict( pms=SensorReader("PMSx003", "/dev/ttyUSB0", interval, samples), bme=SensorReader("MCU680", "/dev/ttyUSB1", interval, samples), ) # read from each sensor and write to DB with measurements_db() as db, reader["pms"] as pms, reader["bme"] as bme: # read one obs from each sensor at the time with progressbar(zip(pms(), bme()), length=samples, label="reading sensors") as progress: for pms_obs, env_obs in progress: write_measurements(db, pms.sensor, pms_obs) write_measurements(db, bme.sensor, env_obs) # read all measurements on the DB and reconstruct sensor.Data objects with measurements_db() as db: # extract obs from one sensor at the time for sensor in [r.sensor for r in reader.values()]: print(sensor) for obs in read_obs(db, sensor): print(obs)
def import_(file_path: str = Argument( None, help="the path of the file to import. Only json files are supported at this " "time.", )): """Import an exported json file to the database.""" if os.path.isfile(file_path): with open(file_path) as f: dump_obj = json.loads(f.read()) time_clok_jobs = dump_obj["time_clok_jobs"] time_clok = dump_obj["time_clok"] time_clok_state = dump_obj["time_clok_state"] time_clok_journal = dump_obj["time_clok_journal"] jobs = [] for job in time_clok_jobs: jobs.append(Job(**job)) add_items_to_database(jobs) cloks = [] for clok in time_clok: cloks.append(Clok(**clok)) add_items_to_database(cloks) journals = [] for journal in time_clok_journal: journals.append(Journal(**journal)) add_items_to_database(journals) try: s = State(**time_clok_state[0]) s.save() except IntegrityError: pass else: raise FileNotFoundError(f"'{file_path}' does not exist.")
def grant_read_access( hw_title: str = Argument(default=..., metavar="📝hw", help="target homework to grant access to"), dry: bool = Opt.DRY, yes: bool = Opt.ACCEPT_ALL, ): """Make TAs being able to read all homework repos "team-slug": the field auto generated by github. For example: the team-slug of "2019 Teaching-team" would be "2019_teaching-team". """ DoCheck(gh_config_valid=True) org, team = settings.github.org, settings.grant.reader_team if not (yes or typer.confirm( f"Grant read access of {org}/{hw_title} to {org}/{team}?")): raise typer.Abort() # additional checks print("check reader team exists") print("check have repo with hw")
def build( ctx: Context, name: Optional[str] = Argument(None, help="Name of images to build"), builder_file: Optional[str] = Option( None, "--builder-file", "-b", help="builder.yml file to use. By default files are searched accross repo", ), *, context: Optional[Path] = Option( None, "--context", "-c", help="Docker context to consider when performing build" ), file: Optional[Path] = Option( None, "--dockerfile", "--file", "-f", help="Custom Dockerfile to use when building image", ), tags: Optional[List[str]] = Option( None, "-t", "--tags", help="Custom tags to give to produced image" ), labels: Optional[List[str]] = Option( None, "-l", "--label", help="Additional labels to give to the produced image" ), platforms: Optional[List[str]] = Option( None, "-p", "--platform", help="Platforms to build the image for" ), add_hosts: Optional[List[str]] = Option( None, "-h", "--add-host", help="Add known hosts entries into the generated docker image", ), push: bool = Option(False, "--push", help="Push the image to registry after build"), load: bool = Option( False, "--load", help="Load the image into local docker engine after build" ), dump: bool = Option( False, "--dump", help="Dump the image filesystem into the current directory after build", ), build_args: Optional[List[str]] = Option( None, "--build-args", help="Additional build arguments" ), builder: Optional[str] = Option( None, "--builder", help="Custom builder to use (see --builder option for 'docker buildx build' command)", ), cache: bool = Option( True, "--cache", help="Cache generated layers to speed up build" ), cache_from: Optional[str] = Option( None, "--cache-from", help="Reuse cache from given location. Can be a remote docker image", ), cache_to: Optional[str] = Option( None, "--cache-to", help="Store intermediate layer and produced cache into given destination", ), network: Optional[str] = Option( None, "--network", help="Use a specific network mode during build" ), output: Optional[str] = Option( None, "--output", help="Custom output for 'docker buildx build' command" ), progress: str = Option("auto", "--progress", help="Progress display mode"), pull: bool = Option(False, "--pull", help="Always pull images before build"), secrets: Optional[List[str]] = Option( None, "--secret", help="Secrets to mount during build" ), # Don't know what those two options are for allow: Optional[List[str]] = None, ssh: Optional[str] = None, target: Optional[str] = None, ) -> None: """Build a docker image.""" if builder_file: try: images = [Image.from_file(builder_file)] except ValidationError as err: console.print(err, style="red") exit(1) elif name: catalog = Catalog.from_directory(Path.cwd()) try: images = [next(image for image in catalog.images if image.name == name)] except StopIteration: console.print( f"Build config for image {style_str(name, 'bold')} does not exist", style="red", ) exit(1) else: try: images = Catalog.from_directory(Path.cwd()).images except ValidationError as err: console.print(err, style="red") exit(1) _add_hosts = map_string_to_dict(add_hosts) _labels = map_string_to_dict(labels or []) names = [tag for tag in tags] if tags else [] is_key = True kwargs = {} for extra_arg in ctx.args: if is_key: key = extra_arg if key.startswith("--"): key = key[2:] elif key.startswith("-"): key = key[1:] if "=" in extra_arg: key, value = key.split("=") key = key.replace("-", "_").upper() kwargs[key] = value is_key = True continue else: is_key = False continue key = key.replace("-", "_").upper() kwargs[key] = extra_arg is_key = True _user_build_args = map_string_to_dict(build_args or []) _user_build_args = {**_user_build_args, **kwargs} if dump: _output = {"type": "local", "dest": "."} elif output: _output = map_string_to_dict(output) else: _output = {} if progress.lower() in ("0", "false", "no", "n"): _progress: Union[bool, str] = False else: _progress = progress # TODO: Is there some order ? for image in images: _build_args = { arg.name.upper(): arg.default for arg in image.build.build_args if arg.default } _build_args.update( {key.upper(): value for key, value in _user_build_args.items()} ) _names = names or [image.get_name()] logger.debug(f"Building image {_names[0]} with build arguments: {_build_args}") if len(_names) > 1: for name in _names[1:]: logger.debug(f"Using additional tag: {name}") build_context = BuildContext( context_path=context or image.build.context, add_hosts=_add_hosts or image.build.add_hosts, allow=list(allow or []), build_args=_build_args, builder=builder, cache=cache, cache_from=cache_from, cache_to=cache_to, file=file or image.build.file, labels={**image.labels, **_labels}, load=load, network=network, output=_output, platforms=list(platforms or []) or image.platforms, progress=_progress, pull=pull, push=push, secrets=secrets, ssh=ssh, tags=_names, target=target, ) docker.buildx.build(**build_context.dict())
def main( exp_min: float = Argument(..., help='Minimal exponent'), exp_max: float = Argument(..., help='Maximal exponent'), ei_snapshots: int = Argument( ..., help='Number of snapshots for empirical interpolation.'), ei_size: int = Argument(..., help='Number of interpolation DOFs.'), snapshots: int = Argument( ..., help='Number of snapshots for basis generation.'), rb_size: int = Argument(..., help='Size of the reduced basis.'), cache_region: Choices('none memory disk persistent') = Option( 'disk', help='Name of cache region to use for caching solution snapshots.' ), ei_alg: Choices('ei_greedy deim') = Option( 'ei_greedy', help='Interpolation algorithm to use.'), grid: int = Option(60, help='Use grid with (2*NI)*NI elements.'), grid_type: Choices('rect tria') = Option('rect', help='Type of grid to use.'), initial_data: Choices('sin bump') = Option( 'sin', help='Select the initial data (sin, bump).'), ipython_engines: int = Option( 0, help= 'If positive, the number of IPython cluster engines to use for parallel greedy search. ' 'If zero, no parallelization is performed.'), ipython_profile: str = Option( None, help='IPython profile to use for parallelization.'), lxf_lambda: float = Option( 1., help='Parameter lambda in Lax-Friedrichs flux.'), periodic: bool = Option( True, help ='If not, solve with dirichlet boundary conditions on left and bottom boundary.' ), nt: int = Option(100, help='Number of time steps.'), num_flux: Choices('lax_friedrichs engquist_osher') = Option( 'engquist_osher', help='Numerical flux to use.'), plot_err: bool = Option(False, help='Plot error.'), plot_ei_err: bool = Option(False, help='Plot empirical interpolation error.'), plot_error_landscape: bool = Option( False, help='Calculate and show plot of reduction error vs. basis sizes.' ), plot_error_landscape_M: int = Option( 10, help='Number of collateral basis sizes to test.'), plot_error_landscape_N: int = Option( 10, help='Number of basis sizes to test.'), plot_solutions: bool = Option(False, help='Plot some example solutions.'), test: int = Option( 10, help='Number of snapshots to use for stochastic error estimation.' ), vx: float = Option(1., help='Speed in x-direction.'), vy: float = Option(1., help='Speed in y-direction.'), ): """Model order reduction of a two-dimensional Burgers-type equation (see pymor.analyticalproblems.burgers) using the reduced basis method with empirical operator interpolation. """ print('Setup Problem ...') problem = burgers_problem_2d(vx=vx, vy=vy, initial_data_type=initial_data.value, parameter_range=(exp_min, exp_max), torus=periodic) print('Discretize ...') if grid_type == 'rect': grid *= 1. / math.sqrt(2) fom, _ = discretize_instationary_fv( problem, diameter=1. / grid, grid_type=RectGrid if grid_type == 'rect' else TriaGrid, num_flux=num_flux.value, lxf_lambda=lxf_lambda, nt=nt) if cache_region != 'none': # building a cache_id is only needed for persistent CacheRegions cache_id = ( f"pymordemos.burgers_ei {vx} {vy} {initial_data}" f"{periodic} {grid} {grid_type} {num_flux} {lxf_lambda} {nt}") fom.enable_caching(cache_region.value, cache_id) print(fom.operator.grid) print(f'The parameters are {fom.parameters}') if plot_solutions: print('Showing some solutions') Us = () legend = () for mu in problem.parameter_space.sample_uniformly(4): print(f"Solving for exponent = {mu['exponent']} ... ") sys.stdout.flush() Us = Us + (fom.solve(mu), ) legend = legend + (f"exponent: {mu['exponent']}", ) fom.visualize(Us, legend=legend, title='Detailed Solutions', block=True) pool = new_parallel_pool(ipython_num_engines=ipython_engines, ipython_profile=ipython_profile) eim, ei_data = interpolate_operators( fom, ['operator'], problem.parameter_space.sample_uniformly(ei_snapshots), error_norm=fom.l2_norm, product=fom.l2_product, max_interpolation_dofs=ei_size, alg=ei_alg.value, pool=pool) if plot_ei_err: print('Showing some EI errors') ERRs = () legend = () for mu in problem.parameter_space.sample_randomly(2): print(f"Solving for exponent = \n{mu['exponent']} ... ") sys.stdout.flush() U = fom.solve(mu) U_EI = eim.solve(mu) ERR = U - U_EI ERRs = ERRs + (ERR, ) legend = legend + (f"exponent: {mu['exponent']}", ) print(f'Error: {np.max(fom.l2_norm(ERR))}') fom.visualize(ERRs, legend=legend, title='EI Errors', separate_colorbars=True) print('Showing interpolation DOFs ...') U = np.zeros(U.dim) dofs = eim.operator.interpolation_dofs U[dofs] = np.arange(1, len(dofs) + 1) U[eim.operator.source_dofs] += int(len(dofs) / 2) fom.visualize(fom.solution_space.make_array(U), title='Interpolation DOFs') print('RB generation ...') reductor = InstationaryRBReductor(eim) greedy_data = rb_greedy( fom, reductor, problem.parameter_space.sample_uniformly(snapshots), use_error_estimator=False, error_norm=lambda U: np.max(fom.l2_norm(U)), extension_params={'method': 'pod'}, max_extensions=rb_size, pool=pool) rom = greedy_data['rom'] print('\nSearching for maximum error on random snapshots ...') tic = time.perf_counter() mus = problem.parameter_space.sample_randomly(test) def error_analysis(N, M): print(f'N = {N}, M = {M}: ', end='') rom = reductor.reduce(N) rom = rom.with_(operator=rom.operator.with_cb_dim(M)) l2_err_max = -1 mumax = None for mu in mus: print('.', end='') sys.stdout.flush() u = rom.solve(mu) URB = reductor.reconstruct(u) U = fom.solve(mu) l2_err = np.max(fom.l2_norm(U - URB)) l2_err = np.inf if not np.isfinite(l2_err) else l2_err if l2_err > l2_err_max: l2_err_max = l2_err mumax = mu print() return l2_err_max, mumax error_analysis = np.frompyfunc(error_analysis, 2, 2) real_rb_size = len(reductor.bases['RB']) real_cb_size = len(ei_data['basis']) if plot_error_landscape: N_count = min(real_rb_size - 1, plot_error_landscape_N) M_count = min(real_cb_size - 1, plot_error_landscape_M) Ns = np.linspace(1, real_rb_size, N_count).astype(np.int) Ms = np.linspace(1, real_cb_size, M_count).astype(np.int) else: Ns = np.array([real_rb_size]) Ms = np.array([real_cb_size]) N_grid, M_grid = np.meshgrid(Ns, Ms) errs, err_mus = error_analysis(N_grid, M_grid) errs = errs.astype(np.float) l2_err_max = errs[-1, -1] mumax = err_mus[-1, -1] toc = time.perf_counter() t_est = toc - tic print(''' *** RESULTS *** Problem: parameter range: ({exp_min}, {exp_max}) h: sqrt(2)/{grid} grid-type: {grid_type} initial-data: {initial_data} lxf-lambda: {lxf_lambda} nt: {nt} not-periodic: {periodic} num-flux: {num_flux} (vx, vy): ({vx}, {vy}) Greedy basis generation: number of ei-snapshots: {ei_snapshots} prescribed collateral basis size: {ei_size} actual collateral basis size: {real_cb_size} number of snapshots: {snapshots} prescribed basis size: {rb_size} actual basis size: {real_rb_size} elapsed time: {greedy_data[time]} Stochastic error estimation: number of samples: {test} maximal L2-error: {l2_err_max} (mu = {mumax}) elapsed time: {t_est} '''.format(**locals())) sys.stdout.flush() if plot_error_landscape: import matplotlib.pyplot as plt import mpl_toolkits.mplot3d # NOQA fig = plt.figure() ax = fig.add_subplot(111, projection='3d') # rescale the errors since matplotlib does not support logarithmic scales on 3d plots # https://github.com/matplotlib/matplotlib/issues/209 surf = ax.plot_surface(M_grid, N_grid, np.log(np.minimum(errs, 1)) / np.log(10), rstride=1, cstride=1, cmap='jet') plt.show() if plot_err: U = fom.solve(mumax) URB = reductor.reconstruct(rom.solve(mumax)) fom.visualize( (U, URB, U - URB), legend=('Detailed Solution', 'Reduced Solution', 'Error'), title='Maximum Error Solution', separate_colorbars=True) global test_results test_results = (ei_data, greedy_data)
def main( backend: Choices('pymor fenics') = Argument(..., help='Discretization toolkit to use.'), alg: Choices('greedy adaptive_greedy pod') = Argument(..., help='The model reduction algorithm to use.'), snapshots: int = Argument( ..., help='greedy/pod: number of training set parameters\n\n' 'adaptive_greedy: size of validation set.' ), rbsize: int = Argument(..., help='Size of the reduced basis.'), test: int = Argument(..., help='Number of test parameters for reduction error estimation.'), ): """Reduced basis approximation of the heat equation.""" # discretize ############ if backend == 'pymor': fom = discretize_pymor() elif backend == 'fenics': fom = discretize_fenics() else: raise NotImplementedError parameter_space=fom.parameters.space(1, 100) # select reduction algorithm with error estimator ################################################# coercivity_estimator = ExpressionParameterFunctional('1.', fom.parameters) reductor = ParabolicRBReductor(fom, product=fom.h1_0_semi_product, coercivity_estimator=coercivity_estimator) # generate reduced model ######################## if alg == 'greedy': rom = reduce_greedy(fom, reductor, parameter_space, snapshots, rbsize) elif alg == 'adaptive_greedy': rom = reduce_adaptive_greedy(fom, reductor, parameter_space, snapshots, rbsize) elif alg == 'pod': rom = reduce_pod(fom, reductor, parameter_space, snapshots, rbsize) else: raise NotImplementedError # evaluate the reduction error ############################## results = reduction_error_analysis( rom, fom=fom, reductor=reductor, error_estimator=True, error_norms=[lambda U: DT * np.sqrt(np.sum(fom.h1_0_semi_norm(U)[1:]**2))], error_norm_names=['l^2-h^1'], condition=False, test_mus=parameter_space.sample_randomly(test, seed=999), plot=True ) # show results ############## print(results['summary']) import matplotlib.pyplot as plt plt.show() # write results to disk ####################### from pymor.core.pickle import dump dump(rom, open('reduced_model.out', 'wb')) results.pop('figure') # matplotlib figures cannot be serialized dump(results, open('results.out', 'wb')) # visualize reduction error for worst-approximated mu ##################################################### mumax = results['max_error_mus'][0, -1] U = fom.solve(mumax) U_RB = reductor.reconstruct(rom.solve(mumax)) if backend == 'fenics': # right now the fenics visualizer does not support time trajectories U = U[len(U) - 1].copy() U_RB = U_RB[len(U_RB) - 1].copy() fom.visualize((U, U_RB, U - U_RB), legend=('Detailed Solution', 'Reduced Solution', 'Error'), separate_colorbars=True) return results
def check_env(cmd: str = Argument( metavar="cmd", default=..., help="command name to check", )): """Check the environemnt is okay"""
def main(n: int = Argument(..., min=0, help="The input n of fact(n)")) -> None: """Compute factorial of a given input.""" colorama.init(autoreset=True, strip=False) print(f"fact({colorama.Fore.CYAN}{n}{colorama.Fore.RESET}) = " f"{colorama.Fore.GREEN}{factorial(n)}{colorama.Fore.RESET}")
def main(grid_intervals: int = Argument(..., help='Grid interval count.'), training_samples: int = Argument( ..., help='Number of samples used for training the reduced basis.')): """Example script for solving linear PDE-constrained parameter optimization problems""" fom, mu_bar = create_fom(grid_intervals) parameter_space = fom.parameters.space(0, np.pi) ranges = parameter_space.ranges['diffusion'] initial_guess = fom.parameters.parse([0.25, 0.5]) def fom_objective_functional(mu): return fom.output(mu) def fom_gradient_of_functional(mu): return fom.output_d_mu(fom.parameters.parse(mu), return_array=True, use_adjoint=True) from functools import partial from scipy.optimize import minimize from time import perf_counter opt_fom_minimization_data = { 'num_evals': 0, 'evaluations': [], 'evaluation_points': [], 'time': np.inf } tic = perf_counter() opt_fom_result = minimize(partial(record_results, fom_objective_functional, fom.parameters.parse, opt_fom_minimization_data), initial_guess.to_numpy(), method='L-BFGS-B', jac=fom_gradient_of_functional, bounds=(ranges, ranges), options={'ftol': 1e-15}) opt_fom_minimization_data['time'] = perf_counter() - tic reference_mu = opt_fom_result.x from pymor.algorithms.greedy import rb_greedy from pymor.reductors.coercive import CoerciveRBReductor from pymor.parameters.functionals import MinThetaParameterFunctional coercivity_estimator = MinThetaParameterFunctional( fom.operator.coefficients, mu_bar) training_set = parameter_space.sample_uniformly(training_samples) training_set_simple = [mu['diffusion'] for mu in training_set] RB_reductor = CoerciveRBReductor(fom, product=fom.energy_product, coercivity_estimator=coercivity_estimator) RB_greedy_data = rb_greedy(fom, RB_reductor, training_set, atol=1e-2) rom = RB_greedy_data['rom'] def rom_objective_functional(mu): return rom.output(mu) def rom_gradient_of_functional(mu): return rom.output_d_mu(fom.parameters.parse(mu), return_array=True, use_adjoint=True) opt_rom_minimization_data = { 'num_evals': 0, 'evaluations': [], 'evaluation_points': [], 'time': np.inf, 'offline_time': RB_greedy_data['time'] } tic = perf_counter() opt_rom_result = minimize(partial(record_results, rom_objective_functional, fom.parameters.parse, opt_rom_minimization_data), initial_guess.to_numpy(), method='L-BFGS-B', jac=rom_gradient_of_functional, bounds=(ranges, ranges), options={'ftol': 1e-15}) opt_rom_minimization_data['time'] = perf_counter() - tic print("\nResult of optimization with FOM model and adjoint gradient") report(opt_fom_result, fom.parameters.parse, opt_fom_minimization_data, reference_mu) print("Result of optimization with ROM model and adjoint gradient") report(opt_rom_result, fom.parameters.parse, opt_rom_minimization_data, reference_mu)
def cli( repo_root: str = Argument(..., help="the fullpath to the git repository"), tags_regex: str = Option( "^v[0-9]", help="regex to select tags to show on changelog" ), starting_rev: str = Option( None, help="starting revision (if not set latest tag starting with " "ghtc_changelog_start if exists, else first git commit)", ), remove_duplicates_entries: bool = Option( True, help="if True, remove duplicate entries" ), unreleased: bool = Option( True, help="if True, add a section about unreleased changes" ), override_file: str = Option( ".ghtc_overrides.ini", help="the path/name of the 'commit overrides' file" ), include_type: List[str] = Option( [], help="include (only) given conventional types in changelog (can be used " "multiple times, all types by default), available types: %s" % ALL_TYPES, ), title: str = "CHANGELOG", unreleased_title: str = "[Unreleased]", debug: bool = Option(False, help="add debug values for each changelog entry"), ): overrides = Overrides(override_file) overrides.parse() repo = Repo(repo_root) previous_tag = starting_rev context: Dict[str, Any] = { "TITLE": title, "UNRELEASED_TAG_TIMESTAMP": UNRELEASED_TAG_TIMESTAMP, "TAGS": [], } tags = get_tags(repo, tags_regex) if len(include_type) == 0: # if include_type is empty, we consider we want all types included_cats = [x.name.lower() for x in list(ConventionalCommitType)] else: included_cats = [x.strip().lower() for x in include_type] if unreleased: tags.append(None) for tag in tags: if tag is None: tag_name = unreleased_title tag_date = UNRELEASED_TAG_TIMESTAMP rev = None else: tag_name = tag.name tag_date = tag.object.authored_date rev = tag_name reverted_commits = [] for commit in get_commits_between(repo, previous_tag, rev): reverted_commit = get_reverted_commit(commit) if reverted_commit is not None: reverted_commits.append(reverted_commit) lines: Dict[ConventionalCommitType, List[ChangelogLine]] = {} for commit in get_commits_between(repo, previous_tag, rev): if commit.hexsha in reverted_commits: continue msg: Optional[ConventionalCommitMessage] = None if commit.hexsha in overrides.commits: msg = overrides.commits[commit.hexsha] if msg is None: # ignored message continue else: msg = parse(commit.message) if msg is None: continue cat = msg.type if cat.name.lower() not in included_cats: continue cline = ChangelogLine(msg, commit.hexsha, commit.committed_date) if cat not in lines: lines[cat] = [] if remove_duplicates_entries and cline in lines[cat]: continue lines[cat].insert(0, cline) entry = ChangelogEntryForATag(tag_name, tag_date, lines) if tag is not None or len(lines) > 0: context["TAGS"].append(entry) context["DEBUG"] = debug previous_tag = tag print(render_template(context))
from typer import Argument, Option from .. import settings # Source source = Argument( default=None, help="Data source [default: stdin]", ) type = Option( default=None, help='Specify type e.g. "package"', ) # File path = Option( default=None, help= "Specify the data path explicitly (e.g. you need to use it if your data is JSON)", ) scheme = Option( default=None, help="Specify scheme [default: inferred]", ) format = Option( default=None, help="Specify format [default: inferred]",
def main( subreddit: str = Argument(..., help=HelpMessages.subreddit), output_dir: str = Option("./data/", help=HelpMessages.output_dir), batch_size: int = Option(10, help=HelpMessages.batch_size), laps: int = Option(3, help=HelpMessages.laps), reddit_id: str = Option(..., help=HelpMessages.reddit_id), reddit_secret: str = Option(..., help=HelpMessages.reddit_secret), reddit_username: str = Option(..., help=HelpMessages.reddit_username), utc_after: Optional[str] = Option(None, help=HelpMessages.utc_after), utc_before: Optional[str] = Option(None, help=HelpMessages.utc_before), debug: bool = Option(False, help=HelpMessages.debug), ): """ Download all the submissions and relative comments from a subreddit. """ # Init direction, out_manager = init_locals(debug, output_dir, subreddit, utc_after, utc_before, run_args=locals()) pushshift_api, reddit_api = init_clients(reddit_id, reddit_secret, reddit_username) logger.info(f"Start download: " f"UTC range: [{utc_before}, {utc_after}], " f"direction: `{direction}`, " f"batch size: {batch_size}, " f"total submissions to fetch: {batch_size * laps}") # Start the gathering for lap in range(laps): lap_message = f"Lap {lap}/{laps} completed in ""{minutes:.1f}m | " \ f"[new/tot]: {len(out_manager.comments_list)}/{out_manager.total_comments_counter}" with Timer(text=lap_message, logger=logger.info): # Reset the data already stored out_manager.reset_lists() # Fetch data in the `direction` way submissions_generator = pushshift_api.search_submissions( subreddit=subreddit, limit=batch_size, sort='desc', sort_type='created_utc', after=utc_after if direction == "after" else None, before=utc_before if direction == "before" else None, ) for sub in submissions_generator: # Fetch the submission data submission_fetcher(sub, out_manager) # Fetch the submission's comments comments_fetcher(sub, out_manager, reddit_api) # Calculate the UTC seen range utc_after, utc_before = utc_range_calculator( sub.created_utc, utc_after, utc_before) # Store data (submission and comments) out_manager.store(lap) logger.debug(f"utc_after: {utc_after} , utc_before: {utc_before}") out_manager.store_utc_params(utc_newer=utc_after, utc_older=utc_before) assert utc_before < utc_after, f"utc_before '{utc_before}' should be less than utc_after '{utc_after}'" logger.info( f"Stop download: lap {laps}/{laps} [total]: {out_manager.total_comments_counter}" )