def main():
from my_descriptor import my_desc
from color_classifier import prepare_dataset
from sklearn.metrics import accuracy_score
import pandas as pd
from rich.console import Console
console = Console()
sizes = [8, 16, 32, 64, 96, 128, 256]
data = {"size": [], "acc": []}
X_train, y_train, X_test, y_test = prepare_dataset()
for size in sizes:
with console.status(
f"[bold green] Training classifier with size {size}...") as _:
clf = TemplateClassifier(descriptor=my_desc, args={"size": size})
clf.fit(X_train, y_train)
with console.status(
f"[bold red] Testing classifier with size {size}...") as _:
y_pred = clf.predict(X_test)
acc = accuracy_score(y_test, y_pred)
data["size"].append(size)
data["acc"].append(acc)
print(f"{size} => {acc}")
print()
df = pd.DataFrame(data)
print(df)
def main(args=None):
"""
Main entry point for training.
"""
parser = get_parser()
args = parser.parse_args(args)
console = Console()
# console.print(args)
mngr = DataManager(save_path=DOCKER_DATA_DIR, report_path=REPORT_DIR)
X, y = mngr.train
X_test, _ = mngr.test
# Train pipeline
start = datetime.now()
search = {k: v for k, v in GRID_SEARCH.items() if k in args.model}
trainer = Trainer(X,
y.values.ravel(),
grid=search,
save_path=MODEL_DIR,
pre_processing=PRE_PROCESSING_STEPS)
trainer.train(args.experiment)
time = datetime.now() - start
console.print(f"Training [green]done[/green] in {time}.")
if args.profile:
# Profile the datasets and save to ../reports/
start = datetime.now()
with console.status("Profiling datasets."):
mngr.profile()
time = datetime.now() - start
console.print(f"Profiling [green]done[/green] in {time}.")
console.print("fReports available in ")
if args.out:
# Check if dir exists
if is_dir(args.out) is False:
make_dir(args.out)
# Save the processed dataset to the provided location.
score, params, hash_id = trainer.performance[
trainer.best_classifier__name]
start = datetime.now()
with console.status(f"Saving processed dataset to {args.out}."):
saved_file = f"data_{hash_id}.pkl"
(trainer.best_classifier.named_steps['pre_processing'].
named_steps['feature_engineering'].transform(X).to_csv(
os.path.join(args.out, saved_file)))
time = datetime.now() - start
console.print(f"Profiling [green]done[/green] in {time}.")
console.print(f"Processed dataset saved to {args.out}/{saved_file} ")
def delete_notebook(id):
"""Delete notebook"""
console = Console()
try:
thread = notebookClient.delete_notebook_async(id)
timeout = time.time() + TIMEOUT
with console.status(
"[bold green] Deleting Notebook(id = {} )...".format(id)):
while not thread.ready():
time.sleep(POLLING_INTERVAL)
if time.time() > timeout:
console.print("[bold red] Timeout!")
return
result = thread.get()
result = result.result
console.print("[bold green] Notebook(id = {} ) deleted".format(id))
except ApiException as err:
if err.body is not None:
errbody = json.loads(err.body)
click.echo("[Api Error] {}".format(errbody["message"]))
else:
click.echo("[Api Error] {}".format(err))
def internet_present(exit=False):
console = Console()
domains = [
"https://google.com",
"https://yahoo.com",
"https://bing.com",
"https://www.ecosia.org",
"https://www.wikipedia.org",
]
results = []
with console.status("Checking internet ...", spinner="material"):
for domain in domains:
try:
requests.get(domain)
results.append(1)
except Exception as e:
results.append(0)
if not any(results):
# print('No internet connection')
if exit:
sys.exit()
return False
else:
return True
class TUI:
def __init__(self):
self.console = Console()
self.today = dt.datetime.now().date()
def initUpdate(self):
self.spinner = self.console.status("Download Compiti")
self.spinner.start()
def finishUpdate(self):
self.spinner.stop()
def printCompiti(self, obj):
lastd = ""
for i in obj:
if not (lastd == i["datCompiti"]):
lastd = i["datCompiti"]
istoday = (dt.date.fromisoformat(lastd) - self.today).days == 1
color = "red" if istoday else "blue"
self.console.print("[" + color + " bold]" + i["datCompiti"] +
"[/" + color + " bold]",
justify="center")
self.console.print("[blue bold]---" + i["desMateria"] +
"[/blue bold]")
self.console.print(i["desCompiti"])
self.console.print("\n")
def get_experiment(id):
"""Get experiments"""
console = Console()
try:
thread = experimentClient.get_experiment_async(id)
timeout = time.time() + TIMEOUT
with console.status(
"[bold green] Fetching Experiment(id = {} )...".format(id)):
while not thread.ready():
time.sleep(POLLING_INTERVAL)
if time.time() > timeout:
console.print("[bold red] Timeout!")
return
result = thread.get()
result = result.result
json_data = richJSON.from_data(result)
console.print(Panel(json_data,
title="Experiment(id = {} )".format(id)))
except ApiException as err:
if err.body is not None:
errbody = json.loads(err.body)
click.echo("[Api Error] {}".format(errbody["message"]))
else:
click.echo("[Api Error] {}".format(err))
def summary(display_error):
console = Console()
console.rule()
dirs = DIRS
run_result: dict[str, Result] = {}
with console.status("Running mypy against all koan files ...",
spinner="moon"):
for directory in dirs:
for py_file in sorted(Path(directory).rglob("*.py")):
name = str(py_file)
res = run_mypy(name)
if res.exit_code == 0:
emoji_text = ":thumbsup:"
else:
emoji_text = ":thumbsdown:"
console.print(
f"[bold] Ran mypy in koan file: {name} {emoji_text}")
run_result[name] = res
display_summary(console, run_result)
if display_error:
for file_name, result in run_result.items():
display_result(console, file_name, result, display="error")
console.rule()
def create_issue(
project: str = typer.Option(..., help="project key", case_sensitive=False),
summary: str = typer.Option(..., help="issue summary"),
desc: str = typer.Option(default="", help="issue desc"),
issue_type: IssueType = typer.Option(default=IssueType.TASK, help="issue type", case_sensitive=False),
):
"""
Create jira issue and Assign to me
"""
is_task_completed = False
console = Console()
with console.status("[bold green]Working on tasks..."):
while not is_task_completed:
config = load_config()
jira = login_jira(config)
new_issue = create_jira_issue(
jira=jira,
project=project,
summary=summary,
description=desc,
issue_type=issue_type,
)
assign_issue(jira=jira, issue=new_issue, account_id=jira.myself()["accountId"])
is_task_completed = True
typer.echo(typer.style(f"[{new_issue.key}] created!! 🌮", fg=typer.colors.GREEN, bold=True))
def signup_cli(self):
Rich().rich_print(
"📒 Alright, Alright, Alright. Let's Create an account for you.")
username = Questionary().ask_for_text("👤 Please Enter Your Username: "******"👤 '{username}' Is Already Taken, Please Enter Your Username: "******"🔑 Please Enter Your Password: "******"🔑 Please Enter Your Password Again: ")
if password_confirmation != password:
Rich().rich_print(
"😮 Oh. The Passwords Are Not The Same, Quitting...")
return False
console = Console()
with console.status("[bold green]Signing-Up...") as status:
if Requests().signup_request(username, password):
Rich().rich_print(
"🦄 Yoo hoo, Your Account Is Created And You Are Logged-In Now."
)
else:
Rich().rich_print("😔 Awwww, Sign-Up Failed.")
class FromBuffer:
def __init__(self):
self.compilers = compilers_
self.console = Console()
self.spinners = spinners
self.output_json = {}
def create_buffer(self, language, editor):
"""Validate language entered and create a buffer for the code editor given."""
if language not in lang_names:
self.console.print("Language is not supported!", style="bold red")
Utils.close()
with tempfile.NamedTemporaryFile(suffix="."+lang_names[language]) as tf:
tf.write(b"")
tf.flush()
call([editor, tf.name])
tf.seek(0)
code = tf.read().decode("utf-8")
self.output_json["code"] = code
self.output_json["compiler"] = self.compilers[language]
with self.console.status(
"Compiling", spinner=random.choice(self.spinners)
) as status:
headers = {"Content-type": "application/json"}
data = requests.post(
"https://wandbox.org/api/compile.json",
headers=headers,
data=json.dumps(self.output_json),
).json()
return data
def delete_files(dataset_slug: str,
files: List[str],
skip_user_confirmation: bool = False) -> None:
"""
Deletes the files from the given dataset.
Exits the application if no dataset with the given slug is found or a general error occurs.
Parameters
----------
dataset_slug: str
The dataset's slug.
files: List[str]
The list of filenames to delete.
skip_user_confirmation: bool
If True, skips user confirmation, if False it will prompt the user. Defaults to False.
"""
client: Client = _load_client(dataset_identifier=dataset_slug)
try:
console = Console()
dataset: RemoteDataset = client.get_remote_dataset(
dataset_identifier=dataset_slug)
items: Iterator[DatasetItem] = dataset.fetch_remote_files(
{"filenames": ",".join(files)})
if not skip_user_confirmation and not secure_continue_request():
console.print("Cancelled.")
return
with console.status("[bold red]Deleting files..."):
dataset.delete_items(items)
console.print("[bold green]Files successfully deleted!")
except NotFound as e:
_error(f"No dataset with name '{e.name}'")
except:
_error(f"An error has occurred, please try again later.")
def run():
console = Console()
console.rule("Cartographer")
auth_session = kroger.authorize(console)
with console.status("Setting up...") as status:
status.update("Loading spreadsheet...")
items = sheets.get_items(console)
status.update("Loading model...")
tokenizer, model = heuristics.load_tokenizer_and_model()
console.log("Loaded model.")
status.update("Picking groceries...")
selected_items = {}
for i, (item, count, description) in enumerate(items):
status.update(f"Picking groceries [green]({i}/{len(items)})...")
search_results = kroger.product_search(item, auth_session)
selected = heuristics.select_item(
item,
search_results.json()["data"],
description,
tokenizer,
model,
console,
)
selected_items[selected["upc"]] = count
status.update("Adding to cart...")
kroger.add_to_cart(selected_items, auth_session)
console.log("Done!")
def comments(
id: str,
workspace: str = get_workspace(),
slug: str = get_slug(),
):
"""View comments for PR by ID"""
url = comments_url.format(workspace=workspace, slug=slug, id=id)
console = Console()
with console.status("[bold green]Loading..."):
resp = get(url)
handle_error(resp)
for comment in resp["values"]:
user = comment["user"]["display_name"]
updated = parse_dt(comment["updated_on"])
path = comment["inline"]["path"]
# _from = comment["inline"]["from"] or ""
to = comment["inline"]["to"] or ""
line = f":{to}" if to else ""
deleted = "(Deleted)" if comment["deleted"] else ""
console.print(
f"[bold]{user}[/bold] {path}{line} [dim]{updated}[/dim] {deleted}"
)
markdown = comment["content"]["raw"]
console.print(Padding(Markdown(markdown, code_theme=config.THEME), 1))
def create_serve(model_name: str, model_version: int):
"""Create serve"""
console = Console()
try:
thread = serveClient.create_serve(model_name,
model_version,
async_req=True)
timeout = time.time() + TIMEOUT
with console.status(
f"[bold green] Creating Serve with name: {model_name}, version: {model_version}"
):
while not thread.ready():
time.sleep(POLLING_INTERVAL)
if time.time() > timeout:
console.print("[bold red] Timeout!")
return
result = thread.get()
click.echo(result)
except ApiException as err:
if err.body is not None:
errbody = json.loads(err.body)
click.echo(f"[Api Error] {errbody['message']}")
else:
click.echo(f"[Api Error] {err}")
def create(
reviewers: List[str] = [],
workspace: str = get_workspace(),
slug: str = get_slug(),
title: str = get_last_commit_msg(),
src_branch: str = get_current_branch(),
dst_branch: str = None,
body: str = None,
close: bool = True,
):
"""Create new PR"""
data = {
"title": title,
"source": {"branch": {"name": src_branch}},
"close_source_branch": close,
}
if dst_branch:
data["destination"] = {"branch": {"name": dst_branch}}
if reviewers:
data["reviewers"] = [{"uuid": r} for r in reviewers]
if body:
data["description"] = body
console = Console()
with console.status("[bold green]Loading..."):
run_cmd(["git", "push", "origin", src_branch])
resp = post(prs_url.format(workspace=workspace, slug=slug), data)
handle_error(resp)
table = generate_prs_table([resp])
console.print(table)
print(resp["links"]["html"]["href"])
def deploy_public_sandbox(uses: str):
"""
Deploy a public sandbox to Jina Hub.
:param uses: the executor uses string
:return: the host and port of the sandbox
"""
scheme, name, tag, secret = parse_hub_uri(uses)
payload = {
'name': name,
'tag': tag if tag else 'latest',
'jina': __version__,
}
from rich.progress import Console
import requests
console = Console()
host = None
try:
res = requests.get(
url=get_hubble_url_v2() + '/rpc/sandbox.get',
params=payload,
headers=get_request_header(),
).json()
if res.get('code') == 200:
host = res.get('data', {}).get('host', None)
except Exception:
raise
if host:
return host, 443
with console.status(
f"[bold green]Deploying sandbox for ({name}) since no existing one..."
):
try:
json = requests.post(
url=get_hubble_url_v2() + '/rpc/sandbox.create',
json=payload,
headers=get_request_header(),
).json()
host = json.get('data', {}).get('host', None)
livetime = json.get('data', {}).get('livetime', '15 mins')
if not host:
raise Exception(f'Failed to deploy sandbox: {json}')
console.log(f"Deployment completed: {host}")
console.log(
f"[bold green]This sandbox will be removed when no traffic during {livetime}"
)
except:
console.log("Deployment failed")
raise
return host, 443
def mlp():
from my_descriptor import my_desc
from color_classifier import prepare_dataset
from sklearn.neural_network import MLPClassifier
from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from rich.console import Console
console = Console()
X_train_names, y_train, X_test_names, y_test = prepare_dataset()
X_train = []
X_test = []
with console.status("[bold green] Calculating descriptors...") as _:
for img_path in X_train_names:
desc = my_desc(img_path, 10)
X_train.append(desc)
for img_path in X_test_names:
desc = my_desc(img_path, 10)
X_test.append(desc)
with console.status("[bold green] Training classifiers...") as _:
clf_mlp = MLPClassifier(random_state=1,
max_iter=300,
hidden_layer_sizes=(250,
50)).fit(X_train, y_train)
clf_svm = SVC().fit(X_train, y_train)
clf_lr = LogisticRegression(random_state=0).fit(X_train, y_train)
with console.status("[bold red] Testing classifiers...") as _:
y_pred_mlp = clf_mlp.predict(X_test)
y_pred_svm = clf_svm.predict(X_test)
y_pred_lr = clf_lr.predict(X_test)
acc_mlp = accuracy_score(y_test, y_pred_mlp)
acc_svm = accuracy_score(y_test, y_pred_svm)
acc_lr = accuracy_score(y_test, y_pred_lr)
print(f"Accuracy MLP: {acc_mlp}")
print(f"Accuracy SVM: {acc_svm}")
print(f"Accuracy LR: {acc_lr}")
def fetch_repos(self):
next_page = 1
repositories = []
console = Console()
headers = self.set_headers()
with console.status(
"[bold green]Fetching repositories from github..."):
self.response = requests.request(self.method,
url=self.url,
headers=headers)
self.update_rate_limit_info()
repositories.extend(self.response.json())
console.log(
f"Success - Fetched Repositories (1 - {len(repositories)})")
if 'link' in self.response.headers:
next_page += 1
link_header = self.response.headers['link']
last_page = int(
re.findall(r'&page=(\d+)>; rel="last"', link_header)[0])
with console.status(
f"[bold green]Fetching additional repositories..."):
while next_page <= last_page:
next_url = self.url + f"&page={next_page}"
next_response = requests.request(self.method,
url=next_url,
headers=headers)
self.update_rate_limit_info()
next_repo_array = next_response.json()
repositories.extend(next_response.json())
starting_repo_index = (next_page - 1) * 100
completed_range = f"({starting_repo_index} - {starting_repo_index + len(next_repo_array)})"
time.sleep(.2)
console.log(
f"Success - Fetched Repositories {completed_range}")
next_page += 1
if self.rate_limit_verbose:
self.display_rate_limit_info()
return repositories
def make_spinner():
"""Make Spinner component with a mocked response table."""
console = Console()
response = make_table()
with console.status('Getting data...'):
work()
console.rule("[bold red]API Response")
console.print(response)
async def create(
self,
paths: Optional[List[str]] = None,
id: Optional[Union[str, 'DaemonID']] = None,
complete: bool = False,
*args,
**kwargs,
) -> Optional['DaemonID']:
"""Create a workspace
:param paths: local file/directory paths to be uploaded to workspace, defaults to None
:param id: workspace id (if already known), defaults to None
:param complete: True if complete_path is used (used by JinadRuntime), defaults to False
:param args: additional positional args
:param kwargs: keyword args
:return: workspace id
"""
async with AsyncExitStack() as stack:
console = Console()
status = stack.enter_context(
console.status('Workspace: ...', spinner='earth'))
workspace_id = None
if id:
workspace_id = daemonize(id)
try:
return (workspace_id if await self._get_helper(
id=workspace_id, status=status) else None)
except (TypeError, ValueError):
self._logger.debug('workspace doesn\'t exist, creating..')
status.update('Workspace: Getting files to upload...')
data = (self._files_in(paths=paths,
exitstack=stack,
complete=complete) if paths else None)
status.update('Workspace: Sending request...')
response = await stack.enter_async_context(
aiohttp.request(
method='POST',
url=self.store_api,
params={'id': workspace_id} if workspace_id else None,
data=data,
))
response_json = await response.json()
workspace_id = next(iter(response_json))
if response.status == HTTPStatus.CREATED:
status.update(f'Workspace: {workspace_id} added...')
return (workspace_id if await self.wait(
id=workspace_id, status=status, logs=True) else None)
else:
self._logger.error(
f'{self._kind.title()} creation failed as: {error_msg_from(response_json)}'
)
return None
class FromInput:
def __init__(self):
init_lexers()
self.compilers = compilers_
self.console = Console()
self.output_json = {}
self.spinners = spinners
self.themes = list(get_all_styles()) + schemes
def get_lang(self, language):
"""Get the programming language from argument, close program if language not supported."""
if language not in self.compilers:
self.console.print("Language is not supported!", style="bold red")
Utils.close()
return language
def askinp(self, theme, language):
"""
Make a multiline prompt for code input and send the code to the api.
The compiled output from the api is returned.
"""
language = self.get_lang(language)
self.console.print("Enter your code, (press esc + enter to run)\n",
style="green")
if theme in self.themes:
try:
style = sfpc(get_style_by_name(theme))
except ClassNotFound:
style = scheme_dict[theme]()
else:
style = sfpc(get_style_by_name("monokai"))
code = prompt(
"",
lexer=PygmentsLexer(lexers_dict[language]),
include_default_pygments_style=False,
style=style,
multiline=True,
)
self.output_json["code"] = code
self.output_json["compiler"] = self.compilers[language]
with self.console.status("Compiling",
spinner=random.choice(
self.spinners)) as status:
headers = {"Content-type": "application/json"}
data = requests.post(
"https://wandbox.org/api/compile.json",
headers=headers,
data=json.dumps(self.output_json),
).json()
return data
def list(
workspace: str = get_workspace(),
):
"""
List repositories in current workspace
"""
console = Console()
with console.status("[bold green]Loading..."):
resp = get(repo_url.format(workspace=workspace))
table = generate_repo_table(resp["values"])
console.print(table)
def main() -> None:
console = Console()
with console.status("Fetching pip versions..."):
pip_versions = get_all_pip_versions()
console.log(f"Found {len(pip_versions)} available pip versions.")
console.log(f"Latest version: {max(pip_versions)}")
with console.status("Generating scripts...") as status:
for variant, mapping in populated_script_constraints(SCRIPT_CONSTRAINTS):
status.update(f"Working on [magenta]{variant}")
console.log(f"[magenta]{variant}")
generate_one(variant, mapping, console=console, pip_versions=pip_versions)
if MOVED_SCRIPTS:
console.log("[magenta]Generating 'moved' scripts...")
with console.status("Generating 'moved' scripts...") as status:
for legacy, current in MOVED_SCRIPTS.items():
status.update(f"Working on [magenta]{legacy}")
generate_moved(legacy, console=console, location=current)
def decline(
id: str,
workspace: str = get_workspace(),
slug: str = get_slug(),
):
"""Decline PR by ID"""
url = decline_url.format(workspace=workspace, slug=slug, id=id)
console = Console()
with console.status("[bold green]Loading..."):
resp = post(url)
handle_error(resp)
print(resp["state"].title())
def request_changes(
id: str,
workspace: str = get_workspace(),
slug: str = get_slug(),
):
"""Request changes for PR by ID"""
url = request_changes_url.format(workspace=workspace, slug=slug, id=id)
console = Console()
with console.status("[bold green]Loading..."):
resp = post(url)
handle_error(resp)
print(resp["state"].title())
def checkout(
id: str,
workspace: str = get_workspace(),
slug: str = get_slug(),
):
"""Checkout PR by ID"""
url = pr_url.format(workspace=workspace, slug=slug, id=id)
console = Console()
with console.status("[bold green]Loading..."):
resp = get(url)
branch_name = resp["source"]["branch"]["name"]
run_cmd(["git", "checkout", "-b", branch_name])
run_cmd(["git", "pull", "origin", branch_name])
def terminate_process(pid):
console = Console(emoji=False)
with console.status("Terminating Webhook Proxy..."):
p = psutil.Process(pid)
for _ in range(2):
p.terminate()
sleep(2)
if p.is_running():
p.kill()
else:
return True
with console.status("Waiting for WebHook Proxy to die..."):
_pass = 0
while p.is_running() or _pass < 8:
sleep(1)
if not p.is_running():
return True
_pass += 1
return False
def login_cli(self):
Rich().rich_print(
"📒 Alright, Alright, Alright. Let's Log-In to your account.")
username = Questionary().ask_for_text("👤 Please Enter Your Username: "******"🔑 Please Enter Your Password: "******"[bold green]Loggin in...") as status:
if Requests().login_request(username, password):
Rich().rich_print("🦄 Yoo hoo, You Are Logged-In Now.")
return True
else:
Rich().rich_print("😔 Awwww, Log-In Failed.")
return False
def list(
workspace: str = get_workspace(),
slug: str = get_slug(),
state: State = State.OPEN,
extra: bool = True
):
"""List all PRs"""
console = Console()
with console.status("[bold green]Loading..."):
response = get(
prs_url.format(workspace=workspace, slug=slug), {"state": state}
)
table = generate_prs_table(response["values"], extra=extra)
console.print(table)
def list_experiment():
"""List experiments"""
COLS_TO_SHOW = [
"Name", "Id", "Tags", "Finished Time", "Created Time", "Running Time",
"Status"
]
console = Console()
try:
thread = experimentClient.list_experiments_async()
timeout = time.time() + TIMEOUT
with console.status("[bold green] Fetching Experiments..."):
while not thread.ready():
time.sleep(POLLING_INTERVAL)
if time.time() > timeout:
console.print("[bold red] Timeout!")
return
result = thread.get()
results = result.result
results = list(
map(
lambda r: [
r["spec"]["meta"]["name"],
r["experimentId"],
",".join(r["spec"]["meta"]["tags"]),
r["finishedTime"],
r["createdTime"],
r["runningTime"],
r["status"],
],
results,
))
table = Table(title="List of Experiments")
for col in COLS_TO_SHOW:
table.add_column(col, overflow="fold")
for res in results:
table.add_row(*res)
console.print(table)
except ApiException as err:
if err.body is not None:
errbody = json.loads(err.body)
click.echo("[Api Error] {}".format(errbody["message"]))
else:
click.echo("[Api Error] {}".format(err))
