def load_yaml_config(path: Path):
"""Loads a YAML config file.
Used in the `train` and `data` command groups when creating datasets or running trainings.
Args:
path (pathlib.Path): path to YAML config file
Returns:
dict: loaded config
Raises:
click.exceptions.BadParameter: any error failing load of config file raises this
with an appropriate error message for the particular error
"""
try:
with open(path, 'r') as stream:
return yaml.safe_load(stream)
except FileNotFoundError as e:
hint = 'config, no such file exists'
raise BadParameter(config, ctx=ctx, param=config, param_hint=hint)
except yaml.parser.ParserError as e:
hint = 'config file, formatting of file is invalid'
raise BadParameter(config, ctx=ctx, param=config, param_hint=hint)
except Exception as e:
hint = 'unknown, unknown error occurred with config file.'
print('-- ERROR BELOW --')
print(e)
raise BadParameter(config, ctx=ctx, param=config, param_hint=hint)
def validate_contract_name(_ctx: Context, _param: Any,
value: Optional[str]) -> Optional[str]:
if value is None:
return None
if value in {c.name for c in CONTRACT_LIST}:
return value
raise BadParameter(f"unknown contract name {value}")
def test_convert(self, mock_json, mock_yaml, mock_super):
mock_cred_obj = create_mock_object(
Credentials, ['convert'], ismagic=True)
mock_cred_obj.fail.side_effect = BadParameter('')
mock_super.convert.return_value = None
mock_json.return_value = self.credentials
mock_yaml.return_value = self.credentials
mock_cred_obj.load_json = mock_json
mock_cred_obj.load_yaml = mock_yaml
mock_cred_obj.file_type = 'ivalid'
self.assertRaises(BadParameter, mock_cred_obj.convert,
mock_cred_obj, '', '', '')
mocks = {
'json': mock_json,
'yaml': mock_yaml,
}
for k, v in mocks.iteritems():
mock_cred_obj.file_type = k
mock_cred_obj.parse_credentials.return_value = ('foo', 'bar')
foo, bar = mock_cred_obj.convert(mock_cred_obj, '', '', '')
self.assertEqual(foo, 'foo')
self.assertEqual(bar, 'bar')
mock_cred_obj.parse_credentials.assert_called_with(v.return_value)
def solve(ctx, length, height, silent, profile, **pieces):
""" Solve a puzzle constrained by board dimensions and pieces. """
# Check that at least one piece is provided.
if not sum(pieces.values()):
context = click.get_current_context()
raise BadParameter('No piece provided.', ctx=context, param_hint=[
'--{}'.format(label) for label in PIECE_LABELS])
# Setup the optionnal profiler.
profiler = BProfile('solver-profile.png', enabled=profile)
solver = SolverContext(length, height, **pieces)
logger.info(repr(solver))
logger.info('Searching positions...')
with profiler:
start = time.time()
for result in solver.solve():
if not silent:
click.echo(u'{}'.format(result))
processing_time = time.time() - start
logger.info('{} results found in {:.2f} seconds.'.format(
solver.result_counter, processing_time))
if profile:
logger.info('Execution profile saved at {}'.format(
profiler.output_path))
def run(ctx, step, commit, environment, watch, adhoc, args):
"""
Start an execution of a step.
"""
if step == '--help': # This is slightly weird, but it's because of the nested command thing
click.echo(ctx.get_help(), color=ctx.color)
ctx.exit()
project = get_project(require=True)
if adhoc:
commit = create_adhoc_commit(project)['identifier']
config = project.get_config()
step = match_prefix(config.steps, step)
if not step:
raise BadParameter(
'{step} is not a known step (try one of {steps})'.format(
step=step,
steps=', '.join(
click.style(t, bold=True) for t in sorted(config.steps))))
step = config.steps[step]
rc = RunCommand(project,
step,
commit=commit,
environment=environment,
watch=watch)
with rc.make_context(rc.name, list(args), parent=ctx) as ctx:
return rc.invoke(ctx)
def validate_ticker(ctx, param, value):
try:
if re.fullmatch(r"^[A-Z]{3}$",
value): # Проверяем соответствует ли тикер формату
c = CurrencyCodes()
if c.get_currency_name(
value): # Если можем получить название по тикеру
return value # то выходим, иначе бросаем эксепшн
else:
raise BadParameter('такого тикера нет')
else:
raise BadParameter('тикер не соответствует формату ААА')
except BadParameter as error: # В случае ошибки
click.echo(f'Ошибка: {error}') # выводим ее
value = click.prompt(param.prompt) # и делаем магию с колбэками
return validate_ticker(ctx, param, value) # которую возвращаем обратно
def kms_ctor(self, loader, node):
try:
binary_data = base64.b64decode(node.value)
if self.kms_session is None:
raise BadParameter("could not establish a KMS session")
kms = self.kms_session.client('kms')
meta = kms.decrypt(CiphertextBlob=binary_data)
unencrypted = meta[u'Plaintext'].decode()
return unencrypted
except ClientError as ex:
secho(ex.response['Error']['Code'])
raise BadParameter("could not decode !kms value: {}".format(ex))
except binascii.Error as ex:
secho("Cannot parse b64 blob: {}".format(ex))
raise BadParameter("could not decode !kms value: {}".format(ex))
def parse_valid_org(indx):
from click.exceptions import BadParameter
try:
return user['organizations'][int(indx) - 1]['subdomain']
except:
raise BadParameter("Please enter an integer between 1 and %s" %
(len(user['organizations'])),
ctx=ctx)
def delete(ctx: Context, *, uuid: UUID) -> None:
async def remove_note() -> None:
async with open_transaction(ctx.obj["db_url"]) as db:
await delete_note(db, uuid)
try:
asyncio.run(remove_note())
except UnknownItemError:
raise BadParameter(f"unknown note '{uuid}'", param_hint="uuid")
def convert(self, value, param, ctx):
try:
int(value)
assert len(value) == 6
return value
except Exception:
raise BadParameter(
f"TOTP tokens should be a 6-digit integer. '{value}' was provided."
)
def parse_package(value):
"""
Parse a composer package version string into a ComposerVersion type
"""
try:
composer_version = util.ComposerVersion.from_package_string(value)
except ValueError as err:
raise BadParameter(err)
return composer_version
def cli_core(ctx, debug, optimize, version, share_email, skip_check_upgrade):
logLevel = logging.DEBUG if debug else logging.INFO
ch = logging.StreamHandler()
ch.setLevel(logLevel)
logger.addHandler(ch)
logger.setLevel(logLevel)
if debug:
logger.debug("Enabled debug level")
logger.debug("-------------------")
# After adding the separate command for "cost" (i.e. `isitfit cost analyze`)
# putting a note here to notify user of new usage
# Ideally, this code would be deprecated though
if ctx.invoked_subcommand is None:
# if still used without subcommands, notify user of new usage
#from .cost import analyze as cost_analyze, optimize as cost_optimize
#if optimize:
# ctx.invoke(cost_optimize, filter_tags=filter_tags, n=n)
#else:
# ctx.invoke(cost_analyze, filter_tags=filter_tags)
from click.exceptions import UsageError
if optimize:
raise UsageError(
"As of version 0.11, please use `isitfit cost optimize` instead of `isitfit --optimize`."
)
elif version:
# ctx.invoke(cli_version)
raise UsageError(
"As of version 0.11, please use `isitfit version` instead of `isitfit --version`."
)
else:
raise UsageError(
"As of version 0.11, please use `isitfit cost analyze` instead of `isitfit` to calculate the cost-weighted utilization."
)
# make sure that context is a dict
ctx.ensure_object(dict)
# check if emailing requested
if share_email is not None:
max_n_recipients = 3
if len(share_email) > max_n_recipients:
from click.exceptions import BadParameter
raise BadParameter(
"Maximum allowed number of email recipients is %i. Received %i"
% (max_n_recipients, len(share_email)),
param_hint="--share-email")
ctx.obj['share_email'] = share_email
# check if current version is out-of-date
if not skip_check_upgrade:
from ..utils import prompt_upgrade
is_outdated = prompt_upgrade('isitfit', isitfit_version)
ctx.obj['is_outdated'] = is_outdated
def validate_amount(ctx, param, value):
try:
if re.fullmatch(r"^\d+(\d*|\.?\d+)$", value):
return value
else:
raise BadParameter('некорректное число для перевода')
except BadParameter as error:
click.echo(f'Ошибка: {error}')
value = click.prompt(param.prompt)
return validate_amount(ctx, param, value)
def update(ctx: Context, *, uuid: UUID, title: str, text: str) -> None:
async def change_note() -> Note:
async with open_transaction(ctx.obj["db_url"]) as db:
return await update_note(db, uuid, title, full_text)
full_text = _read_text(text)
try:
asyncio.run(change_note())
except UnknownItemError:
raise BadParameter(f"unknown note '{uuid}'", param_hint="uuid")
def parse_valid_org(indx):
from click.exceptions import BadParameter
try:
org_indx = int(indx) - 1
if org_indx < 0 or org_indx >= len(org_list):
raise ValueError("Value out of range")
return org_list[org_indx]['subdomain']
except:
raise BadParameter("Please enter an integer between 1 and %s" %
(len(org_list)))
def _get_dt_from_magic_time_pair(num, period):
num = int(num)
period = period.lower()
if period == "d":
delta = timedelta(days=num)
elif period == "h":
delta = timedelta(hours=num)
elif period == "m":
delta = timedelta(minutes=num)
else:
raise BadParameter(f"Couldn't parse magic time string: {num}{period}")
return datetime.utcnow() - delta
def _get_dt_from_date_time_pair(date, time):
date_format = "%Y-%m-%d %H:%M:%S"
if time:
time = "{}:{}:{}".format(*time.split(":") + ["00", "00"])
else:
time = "00:00:00"
date_string = f"{date} {time}"
try:
dt = datetime.strptime(date_string, date_format)
except ValueError:
raise BadParameter(f"Unable to parse date string: {date_string}.")
else:
return dt
def match_step(config, step):
if step in config.steps:
return step
step_matches = match_prefix(config.steps, step, return_unique=False)
if not step_matches:
raise BadParameter(
'"{step}" is not a known step (try one of {steps})'.format(
step=step,
steps=', '.join(
click.style(t, bold=True) for t in sorted(config.steps))),
param_hint='step')
if len(step_matches) > 1:
raise BadParameter(
'"{step}" is ambiguous.\nIt matches {matches}.\nKnown steps are {steps}.'
.format(
step=step,
matches=', '.join(
click.style(t, bold=True) for t in sorted(step_matches)),
steps=', '.join(
click.style(t, bold=True) for t in sorted(config.steps)),
),
param_hint='step')
return step_matches[0]
def validate_routes(ctx, param, value):
"""
Validate the route options.
:param ctx: click Context
:param param: click option
:param value: The value to validate, in this case the route tuples.
:raise BadParameter: If the validation fails.
"""
if not value:
raise BadParameter("Define at least one route.")
if len(value) > 9:
raise BadParameter(
"Currently only 1-9 route colors are supported, not {}.".format(
len(value)))
for name, points in value:
if points > 9 or points < 1:
raise BadParameter(
"Route points must be 1-9, not {}.".format(points))
if len(name) > 3:
raise BadParameter(
"Route names have to consist of 1-3 characters, not {}.".
format(len(name)))
return value
def view(ctx: Context, *, uuid: UUID) -> None:
async def read_note() -> Note:
async with open_transaction(ctx.obj["db_url"]) as db:
return await select_note(db, uuid)
try:
note = asyncio.run(read_note())
except UnknownItemError:
raise BadParameter(f"unknown note '{uuid}'", param_hint="uuid")
click.echo(note.title)
click.echo()
click.echo(note.text)
click.echo()
click.echo(f"Last changed: {note.last_changed}")
def runlocal(ctx, num_validators, node_num, seed, nodial):
if not 0 <= node_num < num_validators:
raise BadParameter("Node number must be between 0 and number of validators - 1")
# reduce key derivation iterations
config = ctx.obj['config']
config, account = _configure_node_network(config, num_validators, node_num, seed)
config['p2p']['min_peers'] = 2
if nodial:
config['discovery']['bootstrap_nodes'] = []
config['p2p']['min_peers'] = 0
app = start_app(config, [account])
serve_until_stopped(app)
def get_window_sizes_from_args(window_sizes: List[str]) -> List[WindowSize]:
result = []
errors = []
for window_size in window_sizes:
try:
width, height = window_size.split(",")
result.append(WindowSize(width=int(width), height=int(height)))
except ValueError:
errors.append(
ErrorWrapper(
BadParameter(
message=
f"🔥 `{window_size}` is not a valid window size. Must be of type `1920,1080`"
),
loc="window_size",
))
if errors:
raise ValidationError(errors=errors, model=WindowSize)
return result
def cli_core(ctx, debug, verbose, optimize, version, share_email,
skip_check_upgrade, skip_prompt_email):
# FIXME click bug: `isitfit cost --help` is calling the code in here. Workaround is to check --help
import sys
if '--help' in sys.argv: return
# make sure that context is a dict
ctx.ensure_object(dict)
# set up exception aggregation in sentry.io
from isitfit import sentry_proxy
from isitfit.apiMan import BASE_URL
sp_url = f"{BASE_URL}fwd/sentry"
sentry_proxy.init(dsn=sp_url)
# test exception caught by sentry. FIXME Dont commit this! :D
# 1/0
# usage stats
# https://docs.python.org/3.5/library/string.html#format-string-syntax
from isitfit.utils import ping_matomo, b2l
ping_url = "/?debug={}&verbose={}&share_email={}&skip_check_upgrade={}"
ping_url = ping_url.format(b2l(debug), b2l(verbose),
b2l(len(share_email) > 0),
b2l(skip_check_upgrade))
ping_matomo(ping_url)
# choose log level based on debug and verbose flags
import logging
logLevel = logging.DEBUG if debug else (
logging.INFO if verbose else logging.WARNING)
ch = logging.StreamHandler()
ch.setLevel(logLevel)
logger.addHandler(ch)
logger.setLevel(logLevel)
if debug:
logger.debug("Enabled debug level")
logger.debug("-------------------")
# After adding the separate command for "cost" (i.e. `isitfit cost analyze`)
# putting a note here to notify user of new usage
# Ideally, this code would be deprecated though
if ctx.invoked_subcommand is None:
# if still used without subcommands, notify user of new usage
#from .cost import analyze as cost_analyze, optimize as cost_optimize
#if optimize:
# ctx.invoke(cost_optimize, filter_tags=filter_tags, n=n)
#else:
# ctx.invoke(cost_analyze, filter_tags=filter_tags)
from click.exceptions import UsageError
if optimize:
err_msg = "As of version 0.11, please use `isitfit cost optimize` instead of `isitfit --optimize`."
ping_matomo("/error/UsageError?message=%s" % err_msg)
raise UsageError(err_msg)
elif version:
# ctx.invoke(cli_version)
err_msg = "As of version 0.11, please use `isitfit version` instead of `isitfit --version`."
ping_matomo("/error/UsageError?message=%s" % err_msg)
raise UsageError(err_msg)
else:
err_msg = "As of version 0.11, please use `isitfit cost analyze` instead of `isitfit` to calculate the cost-weighted utilization."
ping_matomo("/error/UsageError?message=%s" % err_msg)
raise UsageError(err_msg)
# check if emailing requested
if share_email is not None:
max_n_recipients = 3
if len(share_email) > max_n_recipients:
err_msg = "Maximum allowed number of email recipients is %i. Received %i" % (
max_n_recipients, len(share_email))
ping_matomo("/error?message=%s" % err_msg)
from click.exceptions import BadParameter
raise BadParameter(err_msg, param_hint="--share-email")
ctx.obj['share_email'] = share_email
# check if current version is out-of-date
if ctx.invoked_subcommand != 'version':
if not skip_check_upgrade:
from ..utils import prompt_upgrade
is_outdated = prompt_upgrade('isitfit', isitfit_version)
ctx.obj['is_outdated'] = is_outdated
if is_outdated:
ping_matomo("/version/prompt_upgrade?is_outdated=%s" %
b2l(is_outdated))
if ctx.invoked_subcommand not in ['version', 'migrations']:
# run silent migrations
from isitfit.migrations.migman import silent_migrate
migname_l = silent_migrate()
if len(migname_l) > 0:
from isitfit.utils import l2s
migname_s = l2s(migname_l)
ping_matomo("/migrations/silent?migname=%s" % (migname_s))
# save `verbose` and `debug` for later tqdm
ctx.obj['debug'] = debug
ctx.obj['verbose'] = verbose
# save skip-prompt-email for later usage
ctx.obj['skip_prompt_email'] = skip_prompt_email
def init_repo(repo_name: str, template: str):
import os
from distutils.dir_util import copy_tree
from pathlib import Path
from colorama import Fore, Style
if not is_valid_name(repo_name):
raise BadParameter(
message=
"Name should be alphanumeric values and underscores but not start with an underscore",
param_hint="PROJECT_DIRECTORY",
)
repo_path = Path(os.path.join(Path.cwd(), repo_name))
repo_path.mkdir(exist_ok=True)
repo_config_path = repo_path / "feature_store.yaml"
if repo_config_path.exists():
new_directory = os.path.relpath(repo_path, os.getcwd())
print(
f"The directory {Style.BRIGHT + Fore.GREEN}{new_directory}{Style.RESET_ALL} contains an existing feature "
f"store repository that may cause a conflict")
print()
sys.exit(1)
# Copy template directory
template_path = str(
Path(Path(__file__).parent / "templates" / template).absolute())
if not os.path.exists(template_path):
raise IOError(f"Could not find template {template}")
copy_tree(template_path, str(repo_path))
# Seed the repository
bootstrap_path = repo_path / "bootstrap.py"
if os.path.exists(bootstrap_path):
import importlib.util
spec = importlib.util.spec_from_file_location("bootstrap",
str(bootstrap_path))
bootstrap = importlib.util.module_from_spec(spec)
assert isinstance(spec.loader, Loader)
spec.loader.exec_module(bootstrap)
bootstrap.bootstrap() # type: ignore
os.remove(bootstrap_path)
# Template the feature_store.yaml file
feature_store_yaml_path = repo_path / "feature_store.yaml"
replace_str_in_file(feature_store_yaml_path, "project: my_project",
f"project: {repo_name}")
# Remove the __pycache__ folder if it exists
import shutil
shutil.rmtree(repo_path / "__pycache__", ignore_errors=True)
import click
click.echo()
click.echo(
f"Creating a new Feast repository in {Style.BRIGHT + Fore.GREEN}{repo_path}{Style.RESET_ALL}."
)
click.echo()
