def validate_username(ctx, param, username):
if len(username) > User.NAME_LENGTH:
raise BadParameter('must not exceed {} characters'.format(
User.NAME_LENGTH))
if not username or not match(username_regex, username):
raise BadParameter('must match {}'.format(username_regex))
return username
def port_callback(ctx: Context, param: Option, value: str | int) -> int | None:
if ctx.get_parameter_source(param.name) == ParameterSource.DEFAULT:
return None
elif isinstance(value, str) and not value.lstrip("-").isdigit():
raise BadParameter(f"{value!r} is not a valid integer.", ctx, param)
elif (value := int(value)) <= 0:
raise BadParameter(f"{value!r} is not a valid port.", ctx, param)
def app_server(ctx: Context, database: Callable[...,
Database], host: str | None,
port: int | None, ssl_cert: Path | None, ssl_key: Path | None,
redirect_http: int | None, auth: str | None, precache: bool):
"""
Start a server at {yellow}HOST{reset}:{yellow}PORT{reset} to navigate the database. The {yellow}--ssl-cert{reset}
and {yellow}--ssl-cert{reset} allow serving with HTTPS. Setting {yellow}--redirect-http{reset} starts the server in
HTTP to HTTPS redirection mode. For more details on usage see
{blue}https://pypi.org/project/{server_name}/{server_version}{reset}.
"""
if ssl_cert and not ssl_key:
raise BadParameter(
f"'--ssl-cert' and '--ssl-key' must be set together.", ctx,
get_param(ctx, "ssl_key"))
elif ssl_key and not ssl_cert:
raise BadParameter(
f"'--ssl-cert' and '--ssl-key' must be set together.", ctx,
get_param(ctx, 'ssl_cert'))
db: Database = database()
db_path: Path = db.path
db.close()
del db
server(db_path,
host=host,
port=port,
ssl_cert=ssl_cert,
ssl_key=ssl_key,
redirect_port=redirect_http,
precache=precache,
authentication=auth)
def database_edit(ctx: Context, database: Callable[..., Database], table: str, _id: str | int, file: TextIO,
submission_file: BytesIO | None, submission_thumbnail: BytesIO | None):
"""
Edit entries and submission files manually using a JSON file. Submission files/thumbnails can be added using the
respective options.
The JSON fields must match the column names of the selected table. For a list of columns for each table, please see
the README at {blue}https://pypi.org/project/{prog_name}/{version}{reset}.
If the {yellow}--submission-file{reset} and/or {yellow}--submission-thumbnail{reset} options are used, the
{yellow}FILE{reset} argument can be omitted.
"""
db: Database = database()
db_table: Table = get_table(db, table)
data: dict = load(file)
file.close()
if not data and table.lower() != db.submissions.name.lower():
raise BadParameter(f"FILE cannot be empty for {table} table.", ctx, get_param(ctx, "file"))
elif not isinstance(data, dict):
raise BadParameter(f"Data must be in JSON object format.", ctx, get_param(ctx, "file"))
add_history(db, ctx, table=table, id=_id, file=file.name,
submission_file=submission_file.name if submission_file else None,
submission_thumbnail=submission_thumbnail.name if submission_thumbnail else None)
if (entry := db_table[_id]) is None:
raise BadParameter(f"No entry with ID {_id} in {table}.", ctx, get_param(ctx, "_id"))
def main(language, program_file, test_cases_in, compiler_args):
program_file = pathlib.Path(program_file).resolve()
if test_cases_in is None:
if program_file.with_suffix(".txt").exists():
test_cases_in = open(program_file.with_suffix(".txt"), 'r')
elif program_file.with_suffix(".json").exists():
test_cases_in = open(program_file.with_suffix(".json"), 'r')
else:
raise BadParameter("Test case file is missing. Please use -t")
test_cases = get_test_cases(test_cases_in)
test_cases_in.close()
if test_cases == "error":
raise BadParameter(
"invalid testcase file extension, should be .json or .txt")
file_to_run = compile_if_needed(program_file, language, compiler_args)
total_cases = len(test_cases)
failed_cases = 0
languages = {
"python": python,
"java": java,
"pypy": pypy,
"exe": exe,
"c": c,
"cpp": cpp
}
for i, test_case in enumerate(test_cases, 1):
input_data = "\n".join(test_case["input"]).encode()
required_output = "\n".join(test_case["output"])
start_time = time()
process = languages[language](file_to_run, input_data)
time_taken = time() - start_time
actual_output = process.stdout.decode().strip().replace('\r', '')
if process.returncode == 0 and required_output == actual_output:
click.echo(
f"Test Case {i} Passed and took {time_taken:.3f} seconds ✅")
else:
failed_cases += 1
click.echo(f"Test Case {i} Failed ❎")
if process.returncode == 0:
click.echo(
f"Required Output:\n{required_output}\n---------------")
click.echo(
f"Actual Output:\n{actual_output}\n---------------\n(took\
{time_taken:.3f} seconds)\n")
else:
click.echo("---------------")
click.echo(f"Runtime Error: (took {time_taken:.3f} seconds)")
click.echo(process.stderr.decode().strip())
click.echo("---------------\n")
if file_to_run != program_file:
os.remove(file_to_run)
if failed_cases == 0:
click.echo("All tests passed successfully. ✅")
else:
click.echo(f"{failed_cases}/{total_cases} Test Cases Failed ❎")
def validate_pfs_options(options):
eth_address = options.get("pathfinding-eth-address")
if options.get("pathfinding-service-address") not in (
"auto", None) and eth_address is None:
raise BadParameter('"--pathfinding-service-address" set manually '
'but no "--pathfinding-eth-address" given.')
if eth_address is not None and not is_checksum_address(eth_address):
raise BadParameter(
'"--pathfinding-eth-address" value is not a valid EIP55 address.')
def validate_rules(ctx, param, config_rules):
"""Callback to validate the provided file with rules.
The rules are loaded from the file provided in *config_rules* parameter and
processed to a nested dictionary. In the first level, the keys of the
dictionary are the labels used in the rules and "fallback" key (if present
in the rules). The second level keys are "title", "text", "label" or "any"
(applicable for non-fallback rules). For these keys, the value is a list of
compiled regular expression objects. These objects should be used to match
the respective part of a GitHub issue.
Args:
ctx: context object obtained from click
param: parameter object obtained from click
config_rules (str): name of the file containing the rules for issue
assignment
Raises:
BadParameter: if the provided file has incorrect configuration format
Returns:
dict: nested dictionary containing loaded issue assignment rules.
"""
try:
rules_parser = configparser.ConfigParser()
rules_parser.optionxform = str
rules_parser.read(config_rules)
if "patterns" not in rules_parser:
raise BadParameter("incorrect configuration format")
rules = dict()
# Handle fallback
if rules_parser.has_section("fallback"):
if rules_parser.has_option("fallback", "label"):
rules["fallback"] = rules_parser["fallback"]["label"]
else:
raise KeyError
# Handle patterns
for key, value in rules_parser["patterns"].items():
rules[key] = dict()
rules[key]["any"] = list()
rules[key]["title"] = list()
rules[key]["text"] = list()
rules[key]["label"] = list()
for line in value.split("\n"):
rule = line.split(":", 1)
if len(rule) == 2:
rules[key][rule[0]].append(
re.compile(rule[1], re.IGNORECASE))
return rules
except (KeyError, re.error):
raise BadParameter("incorrect configuration format")
def __getitem__(self, name):
try:
f = getattr(self, name)
except AttributeError:
raise BadParameter('No such command "%s".' % name)
else:
if isinstance(f, click.Command):
return f
else:
raise BadParameter('No such command "%s".' % name)
def validate_binary_file(path: str):
import os.path
if not os.path.exists(path):
raise BadParameter('Path does not exist: "{}"'.format(path))
if not path.endswith('.bin'):
raise BadParameter(
'Supplied file "{}" does not have ".bin" extension. '
'Make sure you are sending correct file to the device.'.format(
os.path.basename(path)))
return path
def set_value(secret, namespace, value, file, content):
if file is None and content is None:
raise BadParameter("Please specify either --file or CONTENT")
elif file is not None and content is not None:
raise BadParameter("--file and CONTENT are mutually exclusive")
if file is not None:
print_affected(secrets.set_from_file(namespace, secret, value, file))
elif content is not None:
print_affected(secrets.set_from_text(namespace, secret, value,
content))
def _parse_argument(value: str) -> Tuple[str, object]:
match = re.match(r"^([^=]+)=(.*)$", value)
if not match:
raise BadParameter(f"Invalid format argument: {value}")
key = match.group(1)
try:
value = safe_load(match.group(2))
except MarkedYAMLError as error:
raise BadParameter(f"Invalid YAML format: {value}\n{error}")
return key, value
def validate_power_of_2(value: List[str], ctx: typer.Context):
if len(ctx.params["parameters"]) != len(value):
raise BadParameter(
"power of 2 constraint must be specified for each parameter, mismatching list lengths"
+ ", parameter list lenght:"
+ str(len(ctx.params["parameters"]))
+ ", constraint list length: "
+ str(len(value))
)
for i in value:
if i not in {"y", "n"}:
raise BadParameter("Accepted input are only y/n")
return value
def secret(ctx, param, value):
try:
with open(value) as f:
data = json.load(f)
if (not data.get('mnemonic')):
raise BadParameter('mnemonic not found in json')
except FileNotFoundError:
raise BadParameter(f'{value} not found')
except json.decoder.JSONDecodeError:
raise BadParameter(f'{value} not valid json')
return value
def load_json(ctx, param, value):
"""Load json from parameter."""
if value is not None:
try:
return json.loads(value)
except (KeyError, ValueError):
raise BadParameter('Not a valid JSON.')
def validate_int_metrics(value: List[int]):
for i in value:
if i < 0:
raise BadParameter(
"Invalid metric '" + str(i) + ", all parameters must be >= 0"
)
return value
def validate_target_clock(value: float) -> float:
if value > 0:
return value
raise BadParameter(
"Target clock must be positive, invalid target clock input: "
+ str(value)
)
def validate_option_dependencies(
command_function: Union[click.Command, click.Group],
ctx,
cli_params: Dict[str, Any],
option_dependencies: Dict[str, List[Tuple[str, Any]]],
):
paramname_to_param = {param.name: param for param in command_function.params}
for depending_option_name, requirements in option_dependencies.items():
depending_option_name_int = depending_option_name.replace("-", "_")
param = paramname_to_param[depending_option_name_int]
depending_option_value = cli_params[depending_option_name_int]
if depending_option_value is None:
continue
depending_option_value_default = param.get_default(ctx)
if depending_option_value == depending_option_value_default:
# Ignore dependencies for default values
continue
for depended_option_name, depended_option_required_value in requirements:
depended_option_name_int = depended_option_name.replace("-", "_")
depended_option_actual_value = cli_params[depended_option_name_int]
if depended_option_actual_value != depended_option_required_value:
raise BadParameter(
f'This option is only available when option "--{depended_option_name}" '
f'is set to "{depended_option_required_value}". '
f'Current value: "{depended_option_actual_value}"',
ctx,
param,
)
def app_update(context, filename, format, etag, force):
if etag is None and force is False:
raise BadParameter(
'You need to specify an etag or use --force flag to force the update without document version verification',
param_hint='etag')
file_content = filename.read()
if format == 'yaml':
try:
app = yaml.load(file_content)
except Exception as e:
raise ClickException('Invalid YAML file.') from e
else:
try:
app = json.loads(file_content)
except Exception as e:
raise ClickException('Invalid JSON file.') from e
try:
app_id = context.apps.validate_schema(app, True)
if force:
etag = context.apps.retrieve(app_id).get('_etag')
app['_id'] = app_id
result_id = context.apps.update(app, etag)
click.echo("Application update OK - ID : {}".format(result_id))
except Exception as e:
raise ClickException(
'Cannot update your application. API Exception.\n{}'.format(
e)) from e
def unlink(client, name, include, exclude, yes):
"""Remove matching files from a dataset."""
dataset = client.load_dataset(name=name)
if not dataset:
raise BadParameter('Dataset does not exist.')
records = _filter(
client, names=[dataset.name], include=include, exclude=exclude
)
if not yes and records:
prompt_text = (
'You are about to remove '
'following from "{0}" dataset.\n'.format(dataset.name) +
'\n'.join([str(record.full_path)
for record in records]) + '\nDo you wish to continue?'
)
click.confirm(WARNING + prompt_text, abort=True)
if records:
for item in records:
dataset.unlink_file(item.path)
dataset.to_yaml()
click.secho('OK', fg='green')
def url(ctx, param, value):
url = urlparse(value)
if (url.netloc.lower() != 'github.com'):
raise BadParameter('url must be GitHub link')
return value
def main(song_vs_call_file, output_file):
"""Generate the filtered ids JSON file.
Uses the parameters from `params.yaml` and the metadata.csv file to build a JSON list of ids
"""
song_vs_call_file, output_file = Path(song_vs_call_file), Path(output_file)
if not output_file.parent.exists():
raise BadParameter("output_filepath directory must exist.")
# Load DVC Params file
with open(PARAMS_PATH) as params_file:
params_dict = yaml.safe_load(params_file)
random_seed = params_dict["random_seed"]
song_vs_call_ids = pd.read_json(song_vs_call_file).squeeze()
song_vs_call_ids_train, song_vs_call_ids_test = train_test_split(
song_vs_call_ids, random_state=random_seed)
with open(output_file, "w+") as json_file:
json.dump(
{
"train_ids": song_vs_call_ids_train.tolist(),
"test_ids": song_vs_call_ids_test.tolist(),
},
json_file,
indent=2,
)
def validate_auth(ctx, param, config_auth):
"""Callback to validate the provided authentication file.
The function verifies the provided authentication file, especially the
presence of "token" key in the "github" section.
Args:
ctx: context object obtained from click
param: parameter object obtained from click
config_auth (str): name of the file containing configuraion for
authentication
Raises:
BadParameter: if the provided file has incorrect configuration format
Returns:
dict: contents of the "github" section from the provided file
"""
try:
auth_parser = configparser.ConfigParser()
auth_parser.read(config_auth)
if "token" not in auth_parser["github"]:
raise KeyError
return auth_parser["github"]
except KeyError:
raise BadParameter("incorrect configuration format")
def _validate_python_option(input_value):
if int(input_value) not in {1, 2}:
raise BadParameter(
message="invalid choice: {}. (choose from 1, 2)".format(str(input_value))
)
return int(input_value)
def check_if_model_type_valid(ctx, param, value):
''' Check if the value is either default, coco, or trained '''
if not value in ['default', 'coco', 'trained']:
raise BadParameter(
"The model_type must be either default, coco, or trained.", ctx,
param)
return value
def check_if_url_downloadable(ctx, param, value):
''' check_if_url_downloadable and raise error if not '''
if value == None: return value
if not is_downloadable(value):
raise BadParameter("This url can't be downloaded.", ctx, param)
return value
def main(song_vs_call_file, recordings_file, output_file):
"""Generate the filtered audio time series JSON file.
Uses the song_vs_call file to get relevant recording ids,
recordings_file to get corresponding audio files,
and parameters from `params.yaml` for high-pass filter.
"""
# TODO (@merlerker): The params are unused
# # Load DVC Params file
# with open(PARAMS_PATH) as params_file:
# params_dict = yaml.safe_load(params_file)
# filter_order = params_dict["proc"]["audio"]["filter_order"]
# cutoff_freq = params_dict["proc"]["audio"]["cutoff_freq"]
# Load svc file, recordings, and output
song_vs_call_file, recordings_file, output_file = (
Path(song_vs_call_file),
Path(recordings_file),
Path(output_file),
)
if not output_file.parent.exists():
raise BadParameter("output_filepath directory must exist.")
# Get ids of recordings we are using for song vs call task
svc_ids = pd.read_json(song_vs_call_file).squeeze()
# manual_fc_params
# selected_fc_params
# EfficientFCParameters()
# ComprehensiveFCParameters()
# MinimalFCParameters()
X_df = featurize_set(svc_ids, manual_fc_params)
X_df.to_json(output_file, indent=2, orient="columns")
def create_account(username, password, first, last, email, roles):
first_name = first or 'Test'
last_name = last or 'User'
# Make sure no existing user.
person = db.session.query(Person).filter_by(username=username).first()
if person is not None:
raise BadParameter(
f"Already an account with username '{username}'")
fetched_roles = []
for role_name in roles:
role = db.session.query(Role).filter_by(
name_i18n=role_name).first()
if role:
print(f"Attaching role {role_name} to person")
fetched_roles.append(role)
else:
print(
f"Role {role_name} not found in database, not attached to person"
)
# Create the Person; commit to DB so we get ID
person = Person(first_name=first_name,
last_name=last_name,
username=username,
password=password,
email=email,
roles=fetched_roles)
db.session.add(person)
db.session.commit()
print(f"Created {person}")
def main(metadata_file, output_file):
"""Generate the filtered ids JSON file.
Uses the parameters from `params.yaml` and the metadata.csv file to build a JSON list of ids
"""
metadata_file, output_file = Path(metadata_file), Path(output_file)
if not output_file.parent.exists():
raise BadParameter("output_filepath directory must exist.")
# Load DVC Params file
with open(PARAMS_PATH) as params_file:
params_dict = yaml.safe_load(params_file)
unique_birds = params_dict["build"]["filter"]["unique_birds"]
total_seconds = params_dict["build"]["filter"]["total_seconds"]
manual_filter_list = params_dict["build"]["filter"]["manual_removal"]
quality_levels = params_dict["build"]["filter"]["quality_levels"]
filtered_df = read_and_filter_df(metadata_file, unique_birds,
total_seconds, manual_filter_list,
quality_levels)
with open(output_file, "w+") as json_file:
json.dump(filtered_df.id.tolist(), json_file, indent=2)
def track_paths_in_storage(self, *paths):
"""Track paths in the external storage."""
# Calculate which paths can be tracked in lfs
track_paths = []
attrs = self.find_attr(*paths)
for path in paths:
# Do not add files with filter=lfs in .gitattributes
if attrs.get(path, {}).get('filter') == 'lfs':
continue
path = Path(path)
if path.is_dir():
track_paths.append(str(path / '**'))
elif path.suffix != '.ipynb':
# TODO create configurable filter and follow .gitattributes
track_paths.append(str(path))
if track_paths:
try:
call(
self._CMD_STORAGE_TRACK + track_paths,
stdout=PIPE,
stderr=STDOUT,
cwd=str(self.path),
)
except (KeyboardInterrupt, OSError) as e:
raise BadParameter('Couldn\'t run \'git lfs\':\n{0}'.format(e))
def deploy(context, application_id, module, all_modules, strategy,
safe_deploy_strategy, live_logs, output, no_color):
# TODO find a "clicker" way to do this parameter validation
if not module and not all_modules:
raise MissingParameter(
'You must have one (and only one) from --module and --all-modules parameters',
param_hint='module',
param_type='parameter')
if module and all_modules:
raise BadParameter(
'You must have only one from --module and --all-modules parameters',
param_hint='module')
if all_modules:
app = context.apps.retrieve(application_id)
module = [m["name"] for m in app["modules"]]
modules = []
for m in module:
name, rev = m.split(':') if ':' in m else (m, None)
mod = {"name": name}
if rev:
mod["rev"] = rev
modules.append(mod)
try:
job_id = context.jobs.command_deploy(application_id, modules, strategy,
safe_deploy_strategy)
handle_job_creation(context, job_id, live_logs, output, no_color)
except ApiClientException as e:
raise ClickException(e) from e
