def publish(
file_or_dir: Optional[Path] = typer.Argument(None,
help='Model weight files',
exists=True),
architecture: Optional[str] = typer.Option(None,
'-n',
'--name',
help='Architecture'),
framework: Optional[Framework] = typer.Option(None,
'-fw',
'--framework',
help='Framework'),
engine: Optional[Engine] = typer.Option(None,
'-e',
'--engine',
help='Engine'),
version: Optional[int] = typer.Option(None,
'-v',
'--version',
min=1,
help='Version number'),
task: Optional[Task] = typer.Option(None, '-t', '--task', help='Task'),
dataset: Optional[str] = typer.Option(None,
'-d',
'--dataset',
help='Dataset name'),
metric: Dict[Metric, float] = typer.Option(
'{}',
help='Metrics in the form of mapping JSON string. The map type is '
'`Dict[types.models.mlmodel.Metric, float]`. An example is \'{"acc": 0.76}.\'',
),
inputs: List[IOShape] = typer.Option(
[],
'-i',
'--input',
help=
'List of shape definitions for input tensors. An example of one shape definition is '
'\'{"name": "input", "shape": [-1, 3, 224, 224], "dtype": "TYPE_FP32", "format": "FORMAT_NCHW"}\'',
),
outputs: List[IOShape] = typer.Option(
[],
'-o',
'--output',
help=
'List of shape definitions for output tensors. An example of one shape definition is '
'\'{"name": "output", "shape": [-1, 1000], "dtype": "TYPE_FP32"}\'',
),
convert: Optional[bool] = typer.Option(
True,
'-c',
'--convert',
is_flag=True,
help='Convert the model to other possible format.',
),
profile: Optional[bool] = typer.Option(
False,
'-p',
'--profile',
is_flag=True,
help='Profile the published model(s).',
),
yaml_file: Optional[Path] = typer.Option(
None,
'-f',
'--yaml-file',
exists=True,
file_okay=True,
help=
'Path to configuration YAML file. You should either set the `yaml_file` field or fields '
'(`FILE_OR_DIR`, `--name`, `--framework`, `--engine`, `--version`, `--task`, `--dataset`,'
'`--metric`, `--input`, `--output`).'),
):
meta_info = (file_or_dir, architecture, framework, engine, version, task,
dataset, metric, inputs, outputs)
# check either using parameters, or using YAML
if yaml_file and not any(meta_info):
with open(yaml_file) as f:
model_config = yaml.safe_load(f)
try:
model_yaml = MLModelFromYaml.parse_obj(model_config)
except ValidationError as exc:
typer.echo(exc, err=True, color=True)
raise typer.Exit(422)
elif not yaml_file and all(meta_info):
model_yaml = MLModelFromYaml(
weight=file_or_dir,
architecture=architecture,
framework=framework,
engine=engine,
version=version, # noqa
dataset=dataset,
metric=metric,
task=task,
inputs=inputs,
outputs=outputs,
convert=convert,
profile=profile)
else:
typer.echo(
'Incorrect parameter, you should set either YAML_FILE, or all of the (FILE_OR_DIR, --name,'
'--framework, --engine, --version, --task, --dataset, --metric, --input, --output)'
)
raise typer.Exit(422)
# build request parameters
payload = {'convert': model_yaml.convert, 'profile': model_yaml.profile}
data = model_yaml.dict(use_enum_values=True,
exclude_none=True,
exclude={'convert', 'profile', 'weight'})
form_data = {k: str(v) for k, v in data.items()}
file_or_dir = model_yaml.weight
# read weight files
files = list()
key = 'files'
try:
# read weight file
if file_or_dir.is_dir():
for file in filter(Path.is_file, file_or_dir.rglob('*')):
name = Path(file).relative_to(file_or_dir.parent)
files.append(
(key, (str(name), open(file,
'rb'), 'application/example')))
else:
files.append(
(key, (file_or_dir.name, open(file_or_dir,
'rb'), 'application/example')))
with requests.post(
f'{app_settings.api_v1_prefix}/model/',
params=payload,
data=form_data,
files=files,
) as r:
typer.echo(r.json(), color=True)
finally:
for file in files:
file[1][1].close()
def genreport(path: Path = typer.Argument(..., help="Path to riskmap log")):
rrg = RiskmapReportGenerator(path)
rrg.to_excel("output-report.xlsx")
def main(name: str = typer.Argument("Wade Wilson")):
print(f"Hello {name}")
def http_bench(self, url: str = typer.Argument("url", help="url to send")):
"""say hi to request url"""
print(f"Send request to {url}")
asyncio.run(self.async_http_bench(url))
def network(
trace_file: Path = typer.Argument(...,
exists=True,
dir_okay=False,
help=TRACE_FILE_HELP),
area_file: Path = typer.Argument(...,
exists=True,
dir_okay=False,
help=AREA_FILE_HELP),
snap_threshold: float = typer.Option(0.001, help=SNAP_THRESHOLD_HELP),
determine_branches_nodes: bool = typer.Option(
True,
help=
"Whether to determine branches and nodes as part of analysis. Recommended.",
),
name: Optional[str] = typer.Option(
None,
help="Name for Network. Used when saving outputs and as plot titles."),
circular_target_area: bool = typer.Option(
False, help="Is/are target area(s) circles?"),
truncate_traces: bool = typer.Option(
True,
help="Whether to cut traces at target area boundary. Recommended."),
censoring_area: Optional[Path] = typer.Option(
None,
help=
"Path to area data that delineates censored areas within target areas.",
),
branches_output: Optional[Path] = typer.Option(
None, help="Where to save branch data."),
nodes_output: Optional[Path] = typer.Option(
None, help="Where to save node data."),
general_output: Optional[Path] = typer.Option(
None,
help="Where to save general network analysis outputs e.g. plots."),
parameters_output: Optional[Path] = typer.Option(
None, help="Where to save numerical parameter data from analysis."),
):
"""
Analyze the geometry and topology of trace network.
"""
network_name = name if name is not None else area_file.stem
console = Console()
console.print(
Text.assemble("Performing network analysis of ",
(network_name, "bold green"), "."))
network = Network(
trace_gdf=read_geofile(trace_file),
area_gdf=read_geofile(area_file),
snap_threshold=snap_threshold,
determine_branches_nodes=determine_branches_nodes,
name=network_name,
circular_target_area=circular_target_area,
truncate_traces=truncate_traces,
censoring_area=read_geofile(censoring_area)
if censoring_area is not None else gpd.GeoDataFrame(),
)
(
general_output_path,
branches_output_path,
nodes_output_path,
parameters_output_path,
) = default_network_output_paths(
network_name=network_name,
general_output=general_output,
branches_output=branches_output,
nodes_output=nodes_output,
parameters_output=parameters_output,
)
# Save branches and nodes
console.print(
Text.assemble(
"Saving branches to ",
(str(branches_output_path), "bold green"),
" and nodes to ",
(str(nodes_output_path), "bold green"),
".",
))
network.branch_gdf.to_file(branches_output_path, driver="GPKG")
network.node_gdf.to_file(nodes_output_path, driver="GPKG")
console.print(rich_table_from_parameters(network.parameters))
pd.DataFrame([network.numerical_network_description()
]).to_csv(parameters_output_path)
console.print(
Text.assemble(
"Saving extensive network parameter csv to path:\n",
(str(parameters_output_path), "bold green"),
))
# Plot ternary XYI-node proportion plot
fig, _, _ = network.plot_xyi()
save_fig(fig=fig, results_dir=general_output_path, name="xyi_ternary_plot")
# Plot ternary branch proportion plot
fig, _, _ = network.plot_branch()
save_fig(fig=fig,
results_dir=general_output_path,
name="branch_ternary_plot")
# Plot trace azimuth rose plot
_, fig, _ = network.plot_trace_azimuth()
save_fig(fig=fig, results_dir=general_output_path, name="trace_azimuth")
# Plot trace length distribution plot
_, fig, _ = network.plot_trace_lengths()
save_fig(fig=fig,
results_dir=general_output_path,
name="trace_length_distribution")
def main(
models: List[str] = typer.Argument(
...,
help="List of models to target for generation. The tree will be "
"pruned for these models. Specify 'all' to export all schema.org models.",
),
greedy: bool = typer.Option(
False,
"--greedy",
help="Whether to gulp the model tree recursively, meaning models "
"will be gathered from the field specification in other models. "
"This option does nothing if the 'all' wildcard is used (since the whole "
"graph will be included).",
),
skip_black: bool = typer.Option(
False,
"--skip-black",
help="Disable formatting through black. The generated output will be "
"considerably uglier, but it'll be generated faster.",
),
):
"""Generates a single python source file with pydantic models
representing schema.org models.
"""
all_models = "all" in models
registry = Registry(
Path(__file__).parent / "schemaorg-current-http.jsonld",
data_type_map,
data_type_specificity,
prune_to=None if greedy or all_models else models,
)
models = models if not all_models else registry.all_types()
for type_ in models:
registry.load_type(type_)
if registry.missing_types:
print("Types referenced but missing from the vocabulary:",
file=sys.stderr)
print(repr(registry.missing_types), file=sys.stderr)
with open(Path(__file__).parent / "models.py.tpl") as template_file:
template = jinja_environment.from_string(template_file.read())
template_args = dict(
schemaorg_version=os.getenv("SCHEMAORG_VERSION"),
commit=os.getenv("COMMIT"),
typer_version=typer.__version__,
jinja2_version=jinja2.__version__,
black_version=black.__version__,
skip_black=skip_black,
timestamp=datetime.datetime.now(),
models=registry.models(),
enums=registry.enums(),
)
if skip_black:
template.stream(**template_args).dump(sys.stdout)
else:
print(
black.format_str(
template.render(**template_args),
mode=black.Mode(),
))
def main(csv_name: str = typer.Argument('book.csv')):
typer.echo(f"Running Script with - {csv_name}")
csv_reader = CSVParser(csv_name)
array_of_invoices = csv_reader.get_array_of_invoices()
print(array_of_invoices)
def announce_grade(
homework_prefix: str = typer.Argument(
..., help="prefix of the target homework"),
feedback_source_repo: Optional[str] = typer.Option(
None, show_default=True, help="Repo contains students' feedbacks"),
only_id: Optional[str] = typer.Option(default=None,
help="only id to announce"),
token: Optional[str] = opt_github_token,
org: str = opt_gh_org,
dry: bool = typer.Option(
False, "--dry", help="dry run, do not publish result to the remote"),
yes: bool = opt_all_yes,
):
"""Announce student grades to each hw repo"""
ensure_config_exists()
def fallback(val, fallback_value):
return val if val else fallback_value
# Handle default value manually because we'll change our config after app starts up
token: str = fallback(token,
app_context.config.github.personal_access_token)
org: str = fallback(org, app_context.config.github.organization)
feedback_source_repo: str = fallback(
feedback_source_repo,
app_context.config.announce_grade.feedback_source_repo)
ensure_gh_token(token)
if not (yes or
typer.confirm(f"Add annouce_grade to {org}/{homework_prefix}?")):
raise typer.Abort()
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
student_feedback_title = f"Grade for {homework_prefix}"
gstudents = Gstudents()
feedback_vars = gstudents.left_join(homework_prefix)
# Clone feedback repo & set needed variables
cur = Path(".")
for d in cur.glob("feedback-tmp-*"):
shutil.rmtree(d)
spinner.info("delete dated folder")
root_folder = Path(
tempfile.mkdtemp(
prefix="feedback-tmp-{}-".format(
datetime.now().strftime("%b%d%H%M%S")),
dir=".",
))
spinner.succeed(f"Create tmp folder {root_folder}")
feedback_repo_path = root_folder / "feedbacks"
spinner.info(f"cloning feeback source repo : {feedback_source_repo}")
_, t = measure_time(sp.run)(
[
"git",
"clone",
f"https://github.com/{org}/{feedback_source_repo}.git",
feedback_repo_path.name,
],
cwd=root_folder,
)
spinner.succeed(
f"cloning feeback source repo : {feedback_source_repo} ... {t:4.2f} sec"
)
client = httpx.AsyncClient(headers=httpx.Headers(
{
"User-Agent": "GitHubClassroomUtils/1.0",
"Authorization": "token " + token,
# needed for the check-suites request
"Accept": "application/vnd.github.antiope-preview+json",
}))
hw_path = feedback_repo_path / homework_prefix / "reports"
# generate feedbacks
fbs, t = measure_time(gen_feedbacks)(homework_prefix, hw_path,
feedback_vars)
spinner.succeed(f"Generate content for feedbacks ... {t:5.3f} sec")
# handle only_id
if only_id:
try:
# detect possible buggy condition
info = gstudents.get_student(only_id)
except RuntimeError as e:
print(" *=" * 30)
print("Warning!")
print(e)
return
only_repo_name = get_hw_repo_name(homework_prefix,
info["github_handle"])
fbs = list(filter(lambda fb: fb["repo_name"] == only_repo_name, fbs))
async def push_to_remote(feedback_title, feedbacks):
# push to remote
async def push_feedback(fb):
request_body = {"title": feedback_title, "body": fb["value"]}
try:
issue_num = await find_existing_issue(client, org,
fb["repo_name"],
feedback_title)
except BaseException:
print(f'error on {fb["repo_name"]}')
return
if issue_num:
request_body["state"] = "open" # reopen issue
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues/{issue_num}"
await edit_issue_async(client, url, issue_num, request_body)
else:
url = f"https://api.github.com/repos/{org}/{fb['repo_name']}/issues"
await create_issue_async(client, url, request_body)
print(f'success {fb["repo_name"]}')
async with trio.open_nursery() as nursery:
for fb in feedbacks:
nursery.start_soon(push_feedback, fb)
# print out target repos
print("repo to announce grade:")
pprint([fb["repo_name"] for fb in fbs])
if dry:
spinner.succeed("DRYRUN: skip push to remote")
else:
if typer.confirm("Do you want to continue?", default=False):
_, t = measure_time(trio.run)(push_to_remote,
student_feedback_title, fbs)
spinner.succeed(f"Push feedbacks to remote ... {t:5.2f} sec")
else:
spinner.warn("You refused to publish to remote")
spinner.succeed("finished announce grade")
return
def train_VI(
environment: str = typer.Argument("FrozenLake-v0"),
exploration_steps: int = typer.Option(
100,
show_default=True,
help="Number of random steps to explore at each iteration.",
),
discount_factor: float = typer.Option(
0.9,
show_default=True,
help="Decrease reward of later steps in a trajectory.",
),
batch_size: int = typer.Option(
20,
show_default=True,
help="Number of episodes for testing policy at each iteration.",
),
max_iter: int = typer.Option(
1000,
show_default=True,
help="Maximum number of iterations to train the model for.",
),
reward_objective: float = typer.Option(
0.8,
show_default=True,
help="Objective of reward aaveraged over a test batch to reach.",
),
save: bool = typer.Option(
True,
show_default=True,
help="Save the trained model to disk.",
),
save_name: str = typer.Option(None,
show_default=False,
help="Name for the saved model."),
outdir: Path = typer.Option(
Path.cwd() / "models",
show_default=True,
help=("Output directory for the saving the model " +
"[default: ./models]."),
),
record_gif: bool = typer.Option(
True,
show_default=True,
help="Make a gif of the evolution of the value table.",
),
gif_name: str = typer.Option(
None,
show_default=False,
help="Name for the recorded gif.",
),
record_outdir: Path = typer.Option(
Path.cwd() / "reports/figures",
show_default=True,
help=("Output directory for the saving the gif " +
"[default: ./reports/figures]."),
),
):
"""Train a Value Iteration model by populating value tables.
The model will first randomly explore the environment, populating reward
and transition tables. Then, it will itereate over the various states and
use the Bellman equation and the accumulated information in order to
populate a value table approximating the optimal value function.
Args:
environment (str): name of the Gym environment to train the model on.
Returns:
A defaultdict containing the value table of the environment.
"""
typer.echo(f"Training a value iteration RL model for {environment}")
# Create environment for testing
env = gym.make(environment)
# Initiate Agent
agent = ValueIterationAgent(environment, discount_factor)
if record_gif: snapshots = list()
for iter in range(max_iter):
# Explore environment and populate tables by playing randomly
agent.play_random(exploration_steps)
# Perform the value iteration to populate the value table
agent.value_iteration()
# Use the table as a policy for playing a batch of test episodes
total_reward = 0.0
for episode in range(batch_size):
total_reward += agent.play_episode(env)
average_reward = total_reward / batch_size
typer.echo(f"Iteration: {iter:4d} Reward: {average_reward:7.3f}")
if record_gif:
value_table = agent.get_value_table()
snapshots.append(
draw_vt_heatmap(value_table,
title=f"Value Function, iteration {iter:3d}"))
# Chech for the objective
if average_reward > reward_objective:
typer.echo("Reward objective reached!")
break
typer.echo(f"Process stoped after {iter+1} interations.")
# Save the model as a pickle file, which will hold the dict with the VF
if save:
if save_name is None:
save_name = f"VI_table_{environment}.pkl"
elif save_name[-4] != ".pkl":
save_name += ".pkl"
typer.echo(f"Saving model to {outdir/save_name}")
with open(outdir / save_name, 'wb') as handle:
pickle.dump(agent.get_value_table(),
handle,
protocol=pickle.HIGHEST_PROTOCOL)
# Record a gif of the learning process of the VF over the state space
if record_gif:
if gif_name is None:
gif_name = f"value_function_{environment}.gif"
elif gif_name[-4:] != ".gif":
gif_name += ".gif"
typer.echo(f"Saving gif to {record_outdir/gif_name}")
kwargs_write = {'fps': 1.0, 'quantizer': 'nq'}
imageio.mimsave(record_outdir / gif_name, snapshots, fps=5)
# Also, make and save an image of the best action per state
draw_action_map(agent,
title="Action Map",
save_location=record_outdir /
f"action_map_{environment}.png")
return agent.get_value_table()
def init(dirname: str = typer.Argument(".")):
dirname = Path(dirname)
basemap_zoom = 18 # TODO: Fix
if Path(dirname.joinpath(FILENAME)).exists():
typer.Abort()
if not dirname.is_dir():
typer.Abort()
shortdirname = dirname.resolve().name
uid = inquirer.text(message="Twin ID:", default=shortdirname).execute()
location_keywords = None
location = None
bbox = []
default_name = ""
while True:
location_keywords = inquirer.text(
message="Location keywords (leave blank to skip):",
).execute()
if not location_keywords:
break
geolocator = Nominatim(user_agent="owntwin-cli")
location_candidates = geolocator.geocode(
location_keywords, exactly_one=False, limit=5
)
if not location_candidates:
typer.echo("Not found. Please Try again.")
continue
location = inquirer.select(
message="Select nearby location:",
choices=list(
map(lambda x: {"name": x.address, "value": x}, location_candidates)
)
+ [Separator(), {"name": "(Back)", "value": False}],
).execute()
if location:
break
if not location_keywords:
def validate(text):
try:
return len(tuple(map(lambda x: float(x.strip()), text.split(",")))) == 2
except:
return False
location_coordinate = inquirer.text(
message="Location coordinate [latitude, longitude] (leave blank to skip):",
validate=validate,
filter=lambda text: tuple(map(lambda x: float(x.strip()), text.split(","))),
).execute()
location = namedtuple("location", ["latitude", "longitude", "address"])
location.latitude, location.longitude = location_coordinate
location.address = ""
if location:
lat, lng = location.latitude, location.longitude
# print(location, lat, lng)
size = inquirer.text(
message="Side length (meter):",
filter=lambda x: float(x),
validate=NumberValidator(float_allowed=True),
).execute()
# bbox = [
# lng + utils.meter_to_lng(size, lat, lng), # east
# lat - utils.meter_to_lat(size, lat, lng), # south
# lng - utils.meter_to_lng(size, lat, lng), # west
# lat + utils.meter_to_lat(size, lat, lng), # north
# ]
bbox = [
lng - utils.meter_to_lng(size, lat, lng), # west
lat - utils.meter_to_lat(size, lat, lng), # south
lng + utils.meter_to_lng(size, lat, lng), # east
lat + utils.meter_to_lat(size, lat, lng), # north
]
tiles = mercantile.tiles(*bbox, basemap_zoom)
tiles = list(tiles)
basemap_bbox = utils.tiles_bounds(tiles)
bbox = basemap_bbox
typer.echo(
" Left (lng): {}\n Bottom (lat): {}\n Right (lng): {}\n Top (lat): {}".format(
*bbox
)
)
typer.confirm("Is this OK?", default=True, abort=True)
default_name = location.address
name = inquirer.text(message="Twin name:", default=default_name).execute()
type = inquirer.text(message="Twin type:", default="").execute()
iri = inquirer.text(
message="IRI or URL:", default=f"https://beta.owntwin.com/twin/{uid}"
).execute()
description = inquirer.text(message="Description:", default="").execute()
twin = {
"id": uid,
"name": name,
"type": type,
"type_label": type,
"iri": iri,
"description": description,
"bbox": bbox,
"canvas": {
"width": 1024,
"height": 1024,
},
"terrain": {
"path": "assets/levelmap.json",
},
"building": {
"path": "assets/buildings.json",
},
"modules": {},
}
# twin = {
# "name": name,
# ...
# "terrain": "levelmap.json",
# "objects": {
# "building": "building.json",
# },
# "modules": {},
# }
typer.echo(f"About to create {FILENAME}:\n")
typer.echo(json.dumps(twin, ensure_ascii=False, indent=2))
typer.confirm("Is this OK?", default=True, abort=True)
if not dirname.exists():
dirname.mkdir()
save_config(twin, dirname.joinpath(FILENAME))
def download(
slug: str = typer.Argument(None,
help=ANIME_SLUG_HELP,
callback=select_anime_slug),
directory: str = typer.Option(
CWD_DIR, "--d", help="Directory where files will be uploaded"),
nfrom: int = typer.Option(
None,
"--nfrom",
help="Select episodes greater or equal to the given number"),
nto: int = typer.Option(
None,
"--nto",
help="Select episodes lesser or equal to the given number"),
dfrom: datetime.datetime = typer.Option(
None, "--dfrom", help="Select episodes uploaded after the given date"),
dto: datetime.datetime = typer.Option(
None, "--dto", help="Select episodes uploaded before the given date"),
):
"""
Download a list of episodes from a given anime slug
The output directory can be specified
"""
directory = Path(directory)
animes = get_animes()
animes_by_slug = {anime.slug.slug: anime for anime in animes}
try:
anime = animes_by_slug[slug]
except KeyError:
raise typer.BadParameter(invalid_slug_message(slug=slug,
animes=animes))
sources = get_sources(anime)
filtered_source_ids = set(source.id for source in sources)
if nfrom:
if type(nfrom) is int:
filtered_source_ids -= set(source.id for source in sources
if source.number < nfrom)
if dfrom:
filtered_source_ids -= set(source.id for source in sources
if source.created_at < dfrom)
if nto:
filtered_source_ids -= set(source.id for source in sources
if source.number > nto)
if dto:
filtered_source_ids -= set(source.id for source in sources
if source.created_at > dto)
sources = list(
sorted(
(source for source in sources if source.id in filtered_source_ids),
key=lambda s: s.number,
))
if not sources:
typer.secho("No sources to download ! ", bold=True)
raise typer.Exit(code=1)
typer.echo(download_starting_message(sources=sources, directory=directory))
for source in sources:
typer.echo(
f"{anime.title} - S{anime.season:02d} - E{source.number:02d}")
title_slug = slugify(anime.title)
current_dir = directory / title_slug
if not current_dir.exists():
current_dir.mkdir(parents=True, exist_ok=True)
ext = source.source.rsplit(".", 1)[-1]
filepath = (
current_dir /
f"{title_slug}-S{anime.season:02d}-E{source.number:03d}.{ext}")
download_source(source, filepath)
print_available_field_definitions()
while True:
choice = typer.prompt("Pick one", type=int)
try:
class_name = list(FIELD_GENERATOR_REGISTRY.keys())[choice]
definitions.append((field_name, class_name, []))
break
except IndexError:
typer.echo(f"Error: {choice} is not a valid choice.")
continue
return definitions
app_name_argument = typer.Argument(None)
model_name_argument = typer.Argument(None)
field_definitions_argument = typer.Argument(None)
ordering_option = typer.Option(None, "--ordering", "-o")
django_settings_option = typer.Option(None, "--django-settings", "-s")
def print_code(code):
console = Console()
syntax = Syntax(code, "python")
console.print(syntax)
@app.command(help="Generate a model")
def model(
app_name: Optional[str] = app_name_argument,
def main(name: str = typer.Argument("World", metavar="✨username✨")):
typer.echo(f"Hello {name}")
def main(name: str = typer.Argument("World",
envvar=["AWESOME_NAME", "GOD_NAME"],
show_envvar=True)):
typer.echo(f"Hello {name}")
def build_lang(lang: str = typer.Argument(
..., callback=lang_callback, autocompletion=complete_existing_lang)):
"""
Build the docs for a language, filling missing pages with translation notifications.
"""
lang_path: Path = Path("docs") / lang
if not lang_path.is_dir():
typer.echo(
f"The language translation doesn't seem to exist yet: {lang}")
raise typer.Abort()
typer.echo(f"Building docs for: {lang}")
build_dir_path = Path("docs_build")
build_dir_path.mkdir(exist_ok=True)
build_lang_path = build_dir_path / lang
en_lang_path = Path("docs/en")
site_path = Path("site").absolute()
if lang == "en":
dist_path = site_path
else:
dist_path: Path = site_path / lang
shutil.rmtree(build_lang_path, ignore_errors=True)
shutil.copytree(lang_path, build_lang_path)
en_config_path: Path = en_lang_path / mkdocs_name
en_config: dict = mkdocs.utils.yaml_load(
en_config_path.read_text(encoding="utf-8"))
nav = en_config["nav"]
lang_config_path: Path = lang_path / mkdocs_name
lang_config: dict = mkdocs.utils.yaml_load(
lang_config_path.read_text(encoding="utf-8"))
lang_nav = lang_config["nav"]
# Exclude first 2 entries FastAPI and Languages, for custom handling
use_nav = nav[2:]
lang_use_nav = lang_nav[2:]
file_to_nav = get_file_to_nav_map(use_nav)
sections = get_sections(use_nav)
lang_file_to_nav = get_file_to_nav_map(lang_use_nav)
use_lang_file_to_nav = get_file_to_nav_map(lang_use_nav)
for file in file_to_nav:
file_path = Path(file)
lang_file_path: Path = build_lang_path / "docs" / file_path
en_file_path: Path = en_lang_path / "docs" / file_path
lang_file_path.parent.mkdir(parents=True, exist_ok=True)
if not lang_file_path.is_file():
en_text = en_file_path.read_text(encoding="utf-8")
lang_text = get_text_with_translate_missing(en_text)
lang_file_path.write_text(lang_text, encoding="utf-8")
file_key = file_to_nav[file]
use_lang_file_to_nav[file] = file_key
if file_key:
composite_key = ()
new_key = ()
for key_part in file_key:
composite_key += (key_part, )
key_first_file = sections[composite_key]
if key_first_file in lang_file_to_nav:
new_key = lang_file_to_nav[key_first_file]
else:
new_key += (key_part, )
use_lang_file_to_nav[file] = new_key
key_to_section = {(): []}
for file, file_key in use_lang_file_to_nav.items():
section = get_key_section(key_to_section=key_to_section, key=file_key)
section.append(file)
new_nav = key_to_section[()]
export_lang_nav = [lang_nav[0], nav[1]] + new_nav
lang_config["nav"] = export_lang_nav
build_lang_config_path: Path = build_lang_path / mkdocs_name
build_lang_config_path.write_text(yaml.dump(lang_config,
sort_keys=False,
width=200),
encoding="utf-8")
current_dir = os.getcwd()
os.chdir(build_lang_path)
mkdocs.commands.build.build(
mkdocs.config.load_config(site_dir=str(dist_path)))
os.chdir(current_dir)
typer.secho(f"Successfully built docs for: {lang}",
color=typer.colors.GREEN)
def list(id: int = typer.Argument(...)):
print(tabulate(list(get_file_manifest(jwt().folder(id)))))
def new_lang(lang: str = typer.Argument(..., callback=lang_callback)):
"""
Generate a new docs translation directory for the language LANG.
LANG should be a 2-letter language code, like: en, es, de, pt, etc.
"""
new_path: Path = Path("docs") / lang
if new_path.exists():
typer.echo(f"The language was already created: {lang}")
raise typer.Abort()
new_path.mkdir()
en_docs_path = Path("docs/en")
en_config_path: Path = en_docs_path / mkdocs_name
en_config: dict = mkdocs.utils.yaml_load(
en_config_path.read_text(encoding="utf-8"))
fastapi_url_base = "https://fastapi.tiangolo.com/"
new_config = {}
new_config["site_name"] = en_config["site_name"]
new_config["site_description"] = en_config["site_description"]
new_config["site_url"] = en_config["site_url"] + f"{lang}/"
new_config["theme"] = en_config["theme"]
new_config["theme"]["logo"] = fastapi_url_base + en_config["theme"]["logo"]
new_config["theme"][
"favicon"] = fastapi_url_base + en_config["theme"]["favicon"]
new_config["theme"]["language"] = lang
new_config["repo_name"] = en_config["repo_name"]
new_config["repo_url"] = en_config["repo_url"]
new_config["edit_uri"] = en_config["edit_uri"]
new_config["google_analytics"] = en_config["google_analytics"]
new_config["nav"] = en_config["nav"][:2]
new_config["markdown_extensions"] = en_config["markdown_extensions"]
new_config["extra"] = en_config["extra"]
extra_css = []
css: str
for css in en_config["extra_css"]:
if css.startswith("http"):
extra_css.append(css)
else:
extra_css.append(fastapi_url_base + css)
new_config["extra_css"] = extra_css
extra_js = []
js: str
for js in en_config["extra_javascript"]:
if js.startswith("http"):
extra_js.append(js)
else:
extra_js.append(fastapi_url_base + js)
new_config["extra_javascript"] = extra_js
new_config_path: Path = Path(new_path) / mkdocs_name
new_config_path.write_text(yaml.dump(new_config,
sort_keys=False,
width=200),
encoding="utf-8")
new_config_docs_path: Path = new_path / "docs"
new_config_docs_path.mkdir()
en_index_path: Path = en_docs_path / "docs" / "index.md"
new_index_path: Path = new_config_docs_path / "index.md"
en_index_content = en_index_path.read_text(encoding="utf-8")
new_index_content = f"{missing_translation_snippet}\n\n{en_index_content}"
new_index_path.write_text(new_index_content, encoding="utf-8")
typer.secho(f"Successfully initialized: {new_path}",
color=typer.colors.GREEN)
update_languages(lang=None)
def setup(exercise: str = typer.Argument(None)):
try:
setup_exercise.setup(exercise)
success("Done.")
except setup_exercise.SetupError as e:
error(str(e))
models = []
if file.endswith('.sql'):
models = sql_parser.parse_sql(file)
if file.endswith('.json'):
models = json_parser.parse_json(file)
print(models)
FastApiGenerator.gen_api_files(
models,
templates_path,
targetpath,
project_name,
)
file_arg: str = typer.Argument(
..., help="Path to a SQL upgrade migration or a JSON file")
target_dir_arg: str = typer.Option(".",
help="Path to the target directory",
prompt=True)
project_name_arg: str = typer.Option(...,
help="Your project name fx. 'my_program'")
def main(file: str = file_arg,
target_directory: str = target_dir_arg,
project_name: str = project_name_arg,
git_repo_url: str = typer.Option(None, '--from_repo')):
"""Main entrypoint for the application
"""
def folder(path: Path = typer.Argument(default='.',
exists=True,
file_okay=True,
dir_okay=True,
readable=True,
resolve_path=True),
ignore_extension: str = typer.Option(default=None),
ignore_path: Path = typer.Option(default=None,
exists=True,
file_okay=True,
dir_okay=True,
readable=True,
resolve_path=True),
codec: str = typer.Option(default='hevc')):
"""
Convert all videos and audios in a folder
"""
videos = []
audios = []
for dir, folders, files in os.walk(path):
base_dir = Path(dir)
for item in files:
file_path = base_dir / item
guess = filetype.guess(str(file_path))
if check_ignore(file_path, ignore_extension, ignore_path):
continue
if guess and 'video' in guess.mime:
videos.append(file_path)
if guess and 'audio' in guess.mime:
audios.append(file_path)
manager = enlighten.get_manager()
errors_files = []
pbar = manager.counter(total=len(videos) + len(audios),
desc='Files',
unit='files')
for video in videos:
typer.secho(f'Processing: {video}')
if get_codec(str(video)) != codec:
new_path = convertion_path(video, False)
if new_path.exists():
os.remove(str(new_path))
try:
convert_video_progress_bar(str(video), str(new_path),
choose_encoder(codec), manager)
if new_path.exists() and video.exists and os.stat(
str(new_path)).st_size <= os.stat(str(video)).st_size:
os.remove(str(video))
if video.suffix == new_path.suffix:
shutil.move(new_path, str(video))
else:
if (new_path.exists() and video.exists):
os.remove(str(new_path))
except ffmpeg._run.Error:
typer.secho(f'ffmpeg could not process: {str(video)}', fg=RED)
errors_files.append(video)
pbar.update()
for audio in audios:
typer.secho(f'Processing: {audio}')
if get_codec(str(audio)) != codec:
new_path = convertion_path(audio, True)
if new_path.exists():
os.remove(str(new_path))
try:
convert_file(str(audio), str(new_path), choose_encoder(codec))
if os.stat(str(new_path)).st_size <= os.stat(
str(audio)).st_size:
os.remove(str(audio))
if audio.suffix == new_path.suffix:
shutil.move(new_path, str(audio))
else:
os.remove(str(new_path))
except ffmpeg._run.Error:
typer.secho(
f'ffmpeg could not process this file: {str(audio)}',
fg=RED)
errors_files.append(audio)
pbar.update()
if errors_files:
typer.secho('This files could not be processed:', fg=RED)
typer.secho(str(errors_files), fg=RED)
def main(
noteevents_filepath: str = typer.Argument(
..., help="Filepath to the NOTEEVENTS.csv from MIMIC-III."),
patients_filepath: str = typer.Argument(
..., help="Filepath to the PATIENTS.csv from MIMIC-III."),
output_filepath: str = typer.Argument(
"./train.tsv",
help="Filepath to save the tab-seperated labelled data file."),
num_examples: int = typer.Option(
20000,
help="The number of examples to randomly sample to form the train set."
),
):
typer.secho(
f"Loading data from {noteevents_filepath}...",
bold=True,
)
noteevents = pd.read_csv(
noteevents_filepath,
header=0,
index_col="SUBJECT_ID",
usecols=["SUBJECT_ID", "TEXT", "CATEGORY"],
# Remove whitespace, newlines and tabs from TEXT column data.
converters={"TEXT": lambda text: " ".join(text.strip().split())},
)
# Filter out anything that isn't a discharge summary
noteevents = noteevents[noteevents["CATEGORY"] ==
"Discharge summary"].drop("CATEGORY", axis=1)
typer.secho(
f"Loading data from {patients_filepath}...",
bold=True,
)
patients = pd.read_csv(patients_filepath,
header=0,
index_col="SUBJECT_ID",
usecols=["SUBJECT_ID", "GENDER"])
df = noteevents.join(patients)
output_filepath = Path(output_filepath)
output_filepath.parents[0].mkdir(parents=True, exist_ok=True)
# Sample exactly 1:1 ratio of females to males
f = (df[df["GENDER"] == "F"]).sample(num_examples // 2,
random_state=RANDOM_STATE)
m = (df[df["GENDER"] == "M"]).sample(num_examples // 2,
random_state=RANDOM_STATE)
train_set = pd.concat([f, m]).sample(frac=1, random_state=RANDOM_STATE)
train_set.to_csv(
output_filepath,
sep="\t",
columns=["TEXT", "GENDER"],
header=False,
index=False,
quoting=csv.QUOTE_NONE,
)
typer.secho(
f"Labelled data saved to {output_filepath.absolute()}.",
bold=True,
)
def flow_update(
flow_id: str = typer.Argument(...),
title: str = typer.Option(None, help="The Flow's title."),
definition: str = typer.Option(
None,
help=
("JSON or YAML representation of the Flow to update. May be provided as a filename "
"or a raw string."),
callback=input_validator_callback,
),
subtitle: str = typer.Option(
None,
help="A subtitle for the Flow providing additional, brief description.",
),
description: str = typer.Option(
None, help="A long form description of the Flow's purpose or usage."),
input_schema: str = typer.Option(
None,
help=
("A JSON or YAML representation of a JSON Schema which will be used to "
"validate the input to the deployed Flow when it is run. "
"If not provided, no validation will be performed on Flow input. "
"May be provided as a filename or a raw string."),
callback=input_validator_callback,
),
keywords: List[str] = typer.Option(
None,
"--keyword",
help=
"A keyword which may categorize or help discover the Flow. [repeatable]",
),
visible_to: List[str] = typer.Option(
None,
help=(
"A principal which may see the existence of the deployed Flow. The "
'special value of "public" may be used to control which users can '
"discover this flow. [repeatable]"),
callback=principal_or_public_validator,
),
administered_by: List[str] = typer.Option(
None,
help="A principal which may update the deployed Flow. [repeatable]",
callback=principal_validator,
),
runnable_by: List[str] = typer.Option(
None,
help=
("A principal which may run an instance of the deployed Flow. The special "
'value of "all_authenticated_users" may be used to control which users '
"can invoke this flow. [repeatable]"),
callback=principal_or_all_authenticated_users_validator,
),
validate: bool = typer.Option(
True,
help=
("(EXPERIMENTAL) Perform rudimentary validation of the flow definition."
),
case_sensitive=False,
show_default=True,
),
flows_endpoint: str = typer.Option(
PROD_FLOWS_BASE_URL,
hidden=True,
callback=flows_endpoint_envvar_callback,
),
verbose: bool = verbosity_option,
input_format: InputFormat = typer.Option(
InputFormat.json,
"--input",
"-i",
help="Input format.",
case_sensitive=False,
show_default=True,
),
):
"""
Update a Flow.
"""
fc = create_flows_client(CLIENT_ID, flows_endpoint)
flow_dict = process_input(definition, input_format)
input_schema_dict = process_input(input_schema, input_format,
" for input schema")
result = fc.update_flow(
flow_id,
flow_dict,
title,
subtitle,
description,
keywords,
visible_to,
runnable_by,
administered_by,
input_schema_dict,
validate_definition=validate,
)
if result is not None:
# Match up output format with input format
if input_format is InputFormat.json:
format_and_echo(result, json.dumps, verbose=verbose)
elif input_format is InputFormat.yaml:
format_and_echo(result, yaml.dump, verbose=verbose)
else:
print("No operation to perform")
def tracevalidate(
trace_file: Path = typer.Argument(
...,
exists=True,
file_okay=True,
dir_okay=False,
resolve_path=True,
help=TRACE_FILE_HELP,
),
area_file: Path = typer.Argument(
...,
exists=True,
file_okay=True,
dir_okay=False,
resolve_path=True,
help=AREA_FILE_HELP,
),
allow_fix: bool = typer.Option(
True,
"--allow-fix",
"--fix",
help="Allow the direct modification of trace file to fix errors.",
),
summary: bool = typer.Option(True,
help="Print summary of validation results."),
snap_threshold: float = typer.Option(
0.001,
help=
"Distance threshold used to estimate whether e.g. a trace abuts in another.",
),
output: Optional[Path] = typer.Option(
None, help="Where to save validated output trace data."),
only_area_validation: bool = typer.Option(
False, help="Only validate the area boundary snapping."),
allow_empty_area: bool = typer.Option(
True, help="Allow empty areas to validation."),
):
"""
Validate trace data delineated by target area data.
If allow_fix is True, some automatic fixing will be done to e.g. convert
MultiLineStrings to LineStrings.
"""
console = Console()
# Assert that read files result in GeoDataFrames
traces: gpd.GeoDataFrame = read_geofile(trace_file)
areas: gpd.GeoDataFrame = read_geofile(area_file)
if not all(isinstance(val, gpd.GeoDataFrame) for val in (traces, areas)):
raise TypeError(
"Expected trace and area files to be readable as GeoDataFrames.")
logging.info(f"Validating traces: {trace_file} area: {area_file}.")
# Get input crs
input_crs = traces.crs
# Validate
validation = Validation(
traces,
areas,
trace_file.stem,
allow_fix,
SNAP_THRESHOLD=snap_threshold,
)
if only_area_validation:
console.print(
Text.assemble(("Only performing area validation.", "yellow")))
choose_validators: Optional[Tuple[Type[TargetAreaSnapValidator]]] = (
TargetAreaSnapValidator, )
else:
choose_validators = None
console.print(
Text.assemble("Performing validation of ", (trace_file.name, "blue"),
"."))
validated_trace = validation.run_validation(
choose_validators=choose_validators, allow_empty_area=allow_empty_area)
# Set same crs as input if input had crs
if input_crs is not None:
validated_trace.crs = input_crs
# Get input driver to use as save driver
with fiona.open(trace_file) as open_trace_file:
assert open_trace_file is not None
save_driver = open_trace_file.driver
# Resolve output if not explicitly given
if output is None:
output_dir = make_output_dir(trace_file)
output_path = (trace_file.parent / output_dir /
f"{trace_file.stem}_validated{trace_file.suffix}")
console.print(
Text.assemble((
f"Generated output directory at {output_dir}"
f"\nwhere validated output will be saved at {output_path}.",
"blue",
)))
else:
output_path = output
# Remove file if one exists at output_path
if output_path.exists():
console.print(
Text.assemble(
("Overwriting old file at given output path.", "yellow")))
output_path.unlink()
# Change validation_error column to type: str and consequently save
# the GeoDataFrame.
assert not isinstance(validated_trace[validation.ERROR_COLUMN].iloc[0],
list)
validated_trace.astype({
validation.ERROR_COLUMN: str
}).to_file(output_path, driver=save_driver)
if summary:
describe_results(validated_trace,
validation.ERROR_COLUMN,
console=console)
def flow_action_log(
action_id: str = typer.Argument(...),
flow_id: str = typer.Option(
...,
help="The ID for the Flow which triggered the Action.",
prompt=True,
),
flow_scope: str = typer.Option(
None,
help="The scope this Flow uses to authenticate requests.",
callback=url_validator_callback,
),
reverse: bool = typer.Option(
# Defaulting to any boolean value will reverse output - so we use None
None,
"--reverse",
help=
"Display logs starting from most recent and proceeding in reverse chronological order",
),
limit: int = typer.Option(
None,
help="Set a maximum number of events from the log to return",
min=0,
max=100,
),
marker: str = typer.Option(
None,
"--marker",
"-m",
help="A pagination token for iterating through returned data.",
),
per_page: int = typer.Option(
None,
"--per-page",
"-p",
help=
"The page size to return. Only valid when used without providing a marker.",
min=1,
max=50,
),
output_format: FlowDisplayFormat = typer.Option(
FlowDisplayFormat.json,
"--format",
"-f",
help="Output display format.",
case_sensitive=False,
show_default=True,
),
flows_endpoint: str = typer.Option(
PROD_FLOWS_BASE_URL,
hidden=True,
callback=flows_endpoint_envvar_callback,
),
verbose: bool = verbosity_option,
):
"""
Get a log of the steps executed by a Flow definition's invocation.
"""
fc = create_flows_client(CLIENT_ID, flows_endpoint)
resp = fc.flow_action_log(flow_id, flow_scope, action_id, limit, reverse,
marker, per_page)
if verbose:
display_http_details(resp)
if output_format in (FlowDisplayFormat.json, FlowDisplayFormat.yaml):
_format_and_display_flow(resp, output_format, verbose)
elif output_format in (FlowDisplayFormat.graphviz,
FlowDisplayFormat.image):
flow_def_resp = fc.get_flow(flow_id)
flow_def = flow_def_resp.data["definition"]
colors = state_colors_for_log(resp.data["entries"])
graphviz_out = graphviz_format(flow_def, colors)
if output_format == FlowDisplayFormat.graphviz:
typer.echo(graphviz_out.source)
else:
graphviz_out.render("flows-output/graph", view=True, cleanup=True)
def quickhttp(
directory: Path = typer.Argument(
".", dir_okay=True, file_okay=False, readable=True, help="Directory to serve."
),
timeout: str = typer.Option(
"10m",
"--timeout",
"-t",
help=(
"Time to keep server alive for after most recent request. Accepts time expressions "
"parsable by pytime parse, such as '10m' or '10:00'."
),
),
bind: str = typer.Option(
"127.0.0.1",
"--bind",
"-b",
help=(
"Address to bind server to. '127.0.0.1' (or 'localhost') will only be accessible from "
"this computer. '0.0.0.0' is all interfaces (IP addresses) on this computer, meaning "
"that it can be accessible by other computers at your IP address."
),
),
port: Optional[int] = typer.Option(
None,
"--port",
"-p",
help=(
"Port to use. If None (default), will automatically search for an open port using "
"the other port-related options. If specified, ignores other port-related options."
),
),
port_range_min: int = typer.Option(
DEFAULT_PORT_RANGE_MIN, help="Minimum of range to search for an open port."
),
port_range_max: int = typer.Option(
DEFAULT_PORT_RANGE_MAX, help="Maximum of range to search for an open port."
),
port_max_tries: int = typer.Option(
DEFAULT_PORT_MAX_TRIES, help="Maximum number of ports to check."
),
port_search_type: SearchType = typer.Option(
DEFAULT_PORT_SEARCH_TYPE, help="Type of search to use."
),
version: bool = typer.Option(
False,
"--version",
callback=version_callback,
is_eager=True,
help="Show version and exit.",
show_default=False,
),
):
"""Lightweight CLI that wraps Python's `http.server` with automatic port-finding and shutdown.
"""
timeout_sec = parse(timeout)
if not port:
port = find_available_port(
range_min=port_range_min,
range_max=port_range_max,
max_tries=port_max_tries,
search_type=port_search_type,
)
typer.echo(
f"Starting http.server at http://{bind}:{port} for directory [{directory}]. "
f"Server will stay alive for {str(timedelta(seconds=timeout_sec))}."
)
run_timed_http_server(address=bind, port=port, directory=directory, timeout=timeout_sec)
typer.echo("Server closed.")
def run(
service: str = typer.Argument(
...,
help="Service name",
shell_complete=Application.autocomplete_allservice,
),
pull: bool = typer.Option(
False,
"--pull",
help="Pull the image before starting the container",
show_default=False,
),
debug: bool = typer.Option(
False,
"--debug",
help="Start the container in debug mode",
show_default=False,
),
command: str = typer.Option(
None,
"--command",
help="UNIX command to be executed in the container",
show_default=False,
),
user: str = typer.Option(
None,
"--user",
"-u",
help="User existing in selected service",
show_default=False,
),
first_port: Optional[int] = typer.Option(
None,
"--port",
"-p",
help="port to be associated to the current service interface",
),
detach: Optional[bool] = typer.Option(
None,
"--detach",
help="Start the container in detach mode (default for non-interfaces)",
show_default=False,
),
) -> None:
Application.print_command(
Application.serialize_parameter("--pull", pull, IF=pull),
Application.serialize_parameter("--debug", debug, IF=debug),
Application.serialize_parameter("--command", command, IF=command),
Application.serialize_parameter("--user", user, IF=user),
Application.serialize_parameter("--port", first_port, IF=first_port),
Application.serialize_parameter("", service),
)
Configuration.FORCE_COMPOSE_ENGINE = True
Application.get_controller().controller_init()
Application.get_controller().check_placeholders_and_passwords(
Application.data.compose_config, [service])
if service == REGISTRY and not Configuration.swarm_mode:
print_and_exit("Can't start the registry in compose mode")
docker = Docker()
if Configuration.swarm_mode:
if service != REGISTRY:
docker.registry.ping()
else:
if docker.registry.ping(do_exit=False):
registry = docker.registry.get_host()
print_and_exit("The registry is already running at {}",
registry)
if docker.client.container.exists("registry"):
log.debug(
"The registry container is already existing, removing")
docker.client.container.remove("registry", force=True)
if not debug:
if user:
print_and_exit("Can't specify a user if debug mode is OFF")
if command:
print_and_exit("Can't specify a command if debug mode is OFF")
if user:
log.warning(
"Please remember that users in volatile containers are not mapped on"
" current uid and gid. You should not write or modify files on volumes"
" to prevent permissions errors")
if pull:
log.info("Pulling image for {}...", service)
docker.client.compose.pull([service])
else:
verify_available_images(
[service],
Application.data.compose_config,
Application.data.base_services,
is_run_command=True,
)
# This is equivalent to the old volatile command
if debug:
if not command:
command = "bash"
log.info("Starting {}...", service)
docker.compose.create_volatile_container(
service,
command=command,
user=user,
# if None the wrapper will automatically switch the default ones
# How to prevent ports on volatile containers?
# publish=None,
)
log.info("Service {} removed", service)
return None
# This is equivalent to the old registry command
if service == REGISTRY:
# @ symbol in secrets is not working
# https://github.com/bitnami/charts/issues/1954
# Other symbols like # and " also lead to configuration errors
os.environ["REGISTRY_HTTP_SECRET"] = password(
param_not_used="", length=96
# , symbols="%*,-.=?[]^_~"
)
publish_ports = get_publish_ports(service, first_port)
if detach is None:
if service == "swaggerui" or service == "adminer":
detach = False
else:
detach = True
log.info("Running {}...", service)
if service == "swaggerui":
if Configuration.production:
prot = "https"
else:
prot = "http"
port = publish_ports[0][0] if publish_ports else first_port
log.info(
"You can access SwaggerUI web page here: {}\n",
f"{prot}://{Configuration.hostname}:{port}",
)
if service == "adminer":
if Configuration.production:
prot = "https"
else:
prot = "http"
port = publish_ports[0][0] if publish_ports else first_port
log.info(
"You can access Adminer interface on: {}\n",
f"{prot}://{Configuration.hostname}:{port}",
)
docker.compose.create_volatile_container(service,
detach=detach,
publish=publish_ports)
def viz_prof(profile_report: Path = typer.Argument(
..., help="Path to .prof file")) -> None:
run(f"snakeviz {profile_report}")
def create(
path: str = typer.Argument(
None,
help=("The directory in which the deployment will be created "
"(will be created if does not exist)"),
),
copy_conf: str = typer.Option(
None, help="The configuration to copy from (e.g. dev_config.toml)"),
create_conf: bool = typer.Option(
False,
help="Enable/disable creation of a default configuration file",
),
delete: bool = typer.Option(
False,
help="Delete the the deployment if it exists. "
"Must be specified with --copy-conf or --create-conf",
),
exists_ok: bool = typer.Option(
False, help="Skip the creation if deployment already exists."),
dev: bool = typer.Option(
False,
help=("Enable/disable dev mode "
"(fills the database with test data and allows http access)"),
),
):
"""Create a new Quetz deployment."""
logger.info(f"creating new deployment in path {path}")
deployment_folder = Path(path).resolve()
config_file = deployment_folder / "config.toml"
if _is_deployment(deployment_folder):
if exists_ok:
logger.info(
f'Quetz deployment already exists at {deployment_folder}.\n'
f'Skipping creation.')
return
if delete and (copy_conf or create_conf):
shutil.rmtree(deployment_folder)
else:
typer.echo(
'Use the start command to start a deployment '
'or specify --delete with --copy-conf or --create-conf.',
err=True,
)
raise typer.Abort()
deployment_folder.mkdir(parents=True, exist_ok=True)
# only authorize path with a config file to avoid deletion of unexpected files
# when deleting Quetz instance
if any(f != config_file for f in deployment_folder.iterdir()):
typer.echo(
f'Quetz deployment not allowed at {path}.\n'
'The path should not contain more than the configuration file.',
err=True,
)
raise typer.Abort()
if not config_file.exists() and not create_conf and not copy_conf:
typer.echo(
'No configuration file provided.\n'
'Use --create-conf or --copy-conf to produce a config file.',
err=True,
)
raise typer.Abort()
if copy_conf:
if not os.path.exists(copy_conf):
typer.echo(f'Config file to copy does not exist {copy_conf}.',
err=True)
raise typer.Abort()
typer.echo(f"Copying config file from {copy_conf} to {config_file}")
shutil.copyfile(copy_conf, config_file)
if not config_file.exists() and create_conf:
https = 'false' if dev else 'true'
conf = create_config(https=https)
with open(config_file, 'w') as f:
f.write(conf)
os.environ[_env_prefix + _env_config_file] = str(config_file.resolve())
config = Config(str(config_file))
deployment_folder.joinpath('channels').mkdir(exist_ok=True)
with working_directory(path):
db = get_session(config.sqlalchemy_database_url)
_run_migrations(config.sqlalchemy_database_url)
if dev:
_fill_test_database(db)
_set_user_roles(db, config)
def add_students(
github_students: List[str] = typer.Argument(
metavar="student_handles",
default=...,
help="list of student handles separated by white space",
),
github_team: Optional[str] = typer.Option(
None, "--team", help="invite to the specfic team under organization",
),
yes: bool = opt_all_yes,
dry: bool = opt_dry,
github_token: Optional[str] = opt_github_token,
github_organization: Optional[str] = opt_gh_org,
):
"""
Invite students to join our Github organization
"""
ensure_config_exists()
def fallback(val, fallback_value):
return val if val else fallback_value
# Handle default value manually because we'll change our config after app starts up
github_token: str = fallback(
github_token, app_context.config.github.personal_access_token
)
github_organization: str = fallback(
github_organization, app_context.config.github.organization
)
github_team: str = fallback(
github_team, app_context.config.add_students.default_team_slug
)
safety = SafetyActor(dry=dry)
safety.ensure_gh_token(github_token)
# TODO: use logging lib to log messages
spinner = Halo(stream=sys.stderr)
if dry:
spinner.info("Dry run")
if not (
yes or typer.confirm(f"Add students to {github_organization}/{github_team}?")
):
raise typer.Abort()
spinner.info("fetch existing team members from GitHub")
team = Team(
dry=dry,
org=github_organization,
team_slug=github_team,
github_token=github_token,
)
num_member = len(team.members.keys())
spinner.succeed(f" target team: {github_team} ({num_member} members) ")
existed_members = set(team.members.keys())
outside_users = list(set(github_students) - existed_members)
spinner.info("Check valid Github users")
invalid_handles = invalid_user_handles(
outside_users, github_token=github_token, safety=safety, spinner=spinner
)
if len(invalid_handles) != 0:
print("non-existed github user handles:")
# control strings take space
print_table(invalid_handles)
non_member_valid_users = list(set(outside_users) - set(invalid_handles))
print(f"Users to add (total:{len(non_member_valid_users)})")
print_table(non_member_valid_users)
print("-" * 30)
spinner.info("start to invite users")
success_user, failed_users = invite_user_to_team(
team=team, users=non_member_valid_users, spinner=spinner
)
if len(failed_users) > 0:
print("Users failed to add")
print_table(failed_users)
spinner.succeed("Add students successfully")
def delete(model_id: str = typer.Argument(..., help='Model ID')):
with requests.delete(
f'{app_settings.api_v1_prefix}/model/{model_id}') as r:
if r.status_code == HTTPStatus.NO_CONTENT:
typer.echo(f"Model {model_id} deleted")
