#[Start Global Variables]
FILE_LOCK = False
supported_baudrates = [
50, 75, 110, 134, 150, 200, 300, 600, 1200, 1800, 2400, 4800, 9600, 19200,
38400, 57600, 115200, 230400, 460800, 500000, 576000, 921600, 1000000,
1152000, 1500000, 2000000, 2500000, 3000000, 3500000, 4000000
] #sometimes supported, add as needed.
#[End Global Variables]
# [Start Schemas]
#Define Rules for the JSON Schema to fullfill:
operation_schema = Schema(
{
"startadress":
And(lambda n: (0 <= n <= 9999), int),
"function_code":
And(lambda n: (0 <= n <= 30), int),
"display_name":
str,
"sampling_interval":
Or(And(lambda n: (0.05 <= n <= 864001), (Or(int, float))), 0),
And(lambda n: bool("quantity_of_x" == n) ^ bool("output_value" == n), str):
And(lambda n: (0 <= n <= 500), int
), #Xor: A string: (quantity_of_x Xor output_value) is == integer
},
name="operation_schema",
as_reference=True)
slave_schema = Schema(
{
"task-received": "RECEIVED",
"task-started": "STARTED",
"task-succeeded": "SUCCEEDED",
"task-failed": "FAILED",
"task-rejected": "REJECTED",
"task-revoked": "REVOKED",
"task-retried": "RETRY",
}
TaskEventSchema = Schema(
{
# in case of task-sent, task-received
"type": Or(
*TASK_EVENT_TYPES
),
"uuid": And(str, len),
"timestamp": And(float), # Seconds to Milliseconds
"utcoffset": And(int),
"pid": And(int),
"clock": And(int),
Optional("name"): And(str, len),
Optional("args"): And(str),
Optional("kwargs"): And(str),
# exchange/rq/queue are only available with task-sent events
Optional("exchange"): And(str),
Optional("routing_key"): And(str),
Optional("queue"): And(str),
# If the task has a parent task caller
Optional("root_id"): Or(None, And(str, len)),
Optional("parent_id"): Or(None, And(str, len)),
# countdown: If the task is scheduled or set to be retried after failure
def validate_config(loaded_config):
"""Validate pysmurf configuration dictionary.
Validates the parameters in the configuration dictionary
provided by the `loaded_config` argument using the 3rd party
`schema` python library. If the configuration data is valid,
parameters are returned as a dictionary.
`schema` validation does several important things to raw data
loaded from the pysmurf configuration file:
- Checks that all mandatory configuration variables are defined.
- Conditions all configuration variables into the correct type
(e.g. `float`, `int`, `str`, etc.).
- Automatically fills in the values for missing optional
parameters. Optional parameters are typically parameters which
almost never change from SMuRF system to SMuRF system.
- Checks if parameters have valid values (e.g., some parameters
can only be either 0 or 1, or must be in a predefined interval,
etc.).
- Performs validation of some known higher level configuration
data interdependencies (e.g. prevents the user from defining an
RTM DAC as both a TES bias and an RF amplifier bias).
If validation fails, `schema` will raise a `SchemaError` exception
and fail to load the configuration data, forcing the user to fix
the cause of the `SchemaError` exception before the configuration
file can be loaded and used.
Args
----
loaded_config : dict
Dictionary of pysmurf configuration parameters to run
`schema` validation on.
Returns
-------
validated_config : dict
Dictionary of validated configuration parameters.
Parameter values are conditioned by `schema` to conform to
the specified types.
Raises
------
SchemaError
Raised if the configuration data fails `schema` validation.
"""
# Import useful schema objects
# Try to import them from the system package. If it fails, then
# import the local copy available in this repository.
try:
from schema import Schema, And, Use, Optional, Regex
except ImportError:
from pysmurf.client.base.schema import Schema, And, Use, Optional, Regex
# Start with an extremely limited validation to figure out
# things that we need to validate the entire configuration
# file, like which bands are being used. This also gets used
# to construct the init:bands list, from which band_# are
# present.
band_schema = Schema({'init': {
Regex('band_[0-7]'): {}
}},
ignore_extra_keys=True)
band_validated = band_schema.validate(loaded_config)
# Build list of bands from which band_[0-7] blocks are present. Will
# use this to build the rest of the validation schema.
band_regexp = re.compile("band_([0-7])")
bands = sorted([
int(m.group(1)) for m in (band_regexp.match(s)
for s in band_validated['init'].keys())
if m
])
###################################################
##### Building full validation schema
schema_dict = {}
# Only used if none specified in pysmurf instantiation
schema_dict['epics_root'] = And(str, len)
#### Start specifiying init schema
# init must be present
schema_dict['init'] = {}
# Explicitly sets dspEnable to this value in pysmurf setup. Shouldn't
# be necessary if defaults.yml is properly configured (dspEnable can
# be set there, instead).
schema_dict['init'][Optional('dspEnable',
default=1)] = And(int, lambda n: n in
(0, 1))
# Each band has a configuration block in init.
# Default data_out_mux definitions. Should match fw.
default_data_out_mux_dict = {}
default_data_out_mux_dict[0] = [2, 3]
default_data_out_mux_dict[1] = [0, 1]
default_data_out_mux_dict[2] = [6, 7]
default_data_out_mux_dict[3] = [8, 9]
default_data_out_mux_dict[4] = [2, 3]
default_data_out_mux_dict[5] = [0, 1]
default_data_out_mux_dict[6] = [6, 7]
default_data_out_mux_dict[7] = [8, 9]
for band in bands:
schema_dict['init'][f'band_{band}'] = {
# Swap IQ channels on input
Optional('iq_swap_in', default=0): And(int, lambda n: n in (0, 1)),
# Swap IQ channels on output
Optional('iq_swap_out', default=0): And(int, lambda n: n in (0, 1)),
# Global feedback enable
Optional('feedbackEnable', default=1): And(int, lambda n: n in (0, 1)),
# Global feedback polarity
Optional('feedbackPolarity', default=1): And(int, lambda n: n in (0, 1)),
# Global feedback gain (might no longer be used in dspv3).
"feedbackGain" : And(int, lambda n: 0 <= n < 2**16),
# Global feedback gain (might no longer be used in dspv3).
"feedbackLimitkHz" : And(Use(float), lambda f: f > 0),
# Number of cycles to delay phase reference
'refPhaseDelay': And(int, lambda n: 0 <= n < 2**4),
# Finer phase reference delay, 307.2MHz clock ticks. This
# goes in the opposite direction as refPhaseDelay.
'refPhaseDelayFine': And(int, lambda n: 0 <= n < 2**8),
# RF attenuator on SMuRF output. UC=up convert. 0.5dB steps.
'att_uc': And(int, lambda n: 0 <= n < 2**5),
# RF attenuator on SMuRF input. DC=down convert. 0.5dB steps.
'att_dc': And(int, lambda n: 0 <= n < 2**5),
# Tone amplitude. 3dB steps.
'amplitude_scale': And(int, lambda n: 0 <= n < 2**4),
# data_out_mux
Optional("data_out_mux",
default=default_data_out_mux_dict[band]) : \
And([Use(int)], list, lambda l: len(l) == 2 and
l[0] != l[1] and all(0 <= ll <= 9 for ll in l)),
# Matches system latency for LMS feedback (9.6 MHz
# ticks, use multiples of 52). For dspv3 to adjust to
# match refPhaseDelay*4 (ignore refPhaseDelayFine for
# this). If not provided and lmsDelay=None, sets to
# lmsDelay = 4 x refPhaseDelay.
Optional('lmsDelay', default=None) : And(int, lambda n: 0 <= n < 2**5),
# Adjust trigRstDly such that the ramp resets at the flux ramp
# glitch. 2.4 MHz ticks.
'trigRstDly': And(int, lambda n: 0 <= n < 2**7),
# LMS gain, powers of 2
'lmsGain': And(int, lambda n: 0 <= n < 2**3),
}
#### Done specifying init schema
#### Start specifying attenuator schema
# Here's another one that users probably shouldn't be touching Just
# doing basic validation here - not checking if they're distinct, for
# instance.
schema_dict["attenuator"] = {
'att1': And(int, lambda n: 0 <= n < 4),
'att2': And(int, lambda n: 0 <= n < 4),
'att3': And(int, lambda n: 0 <= n < 4),
'att4': And(int, lambda n: 0 <= n < 4)
}
#### Done specifying attenuator schema
#### Start specifying cryostat card schema
## SHOULD MAKE IT SO THAT WE JUST LOAD AND VALIDATE A SEPARATE,
## HARDWARE SPECIFIC CRYOSTAT CARD CONFIG FILE. FOR NOW, JUST DO
## SOME VERY BASIC VALIDATION.
def represents_int(string):
try:
int(string)
return True
except ValueError:
return False
schema_dict["pic_to_bias_group"] = {And(str, represents_int): int}
schema_dict["bias_group_to_pair"] = {
And(str, represents_int): [int, int]
}
#### Done specifying cryostat card schema
#### Start specifiying amplifier
schema_dict["amplifier"] = {
# 4K amplifier gate voltage, in volts.
"hemt_Vg":
Use(float),
# Conversion from bits (the digital value the RTM DAC is set to)
# to volts for the 4K amplifier gate. Units are volts/bit. An
# important dependency is the voltage division on the cryostat
# card, which can be different from cryostat card to cryostat card
"bit_to_V_hemt":
And(Use(float), lambda f: f > 0),
# The 4K amplifier drain current is measured before a voltage
# regulator, which also draws current. An accurate measurement of
# the 4K drain current requires subtracting the current drawn by
# that regulator. This is the offset to subtract off the measured
# value, in mA.
"hemt_Id_offset":
Use(float),
"50k_Id_offset":
Use(float),
# The resistance, in Ohm, of the resistor that is inline
# with the 4K HEMT amplifier drain voltage source which is
# used to infer the 4K HEMT amplifier drain current. The
# default value of 200 Ohm is the standard value in the
# BOM for cryostat card revision C02 (PC-248-103-02-C02).
# The resistor on that revision of the cryostat card is
# R44.
Optional('hemt_Vd_series_resistor', default=200.0):
And(float, lambda f: f > 0),
# The resistance, in Ohm, of the resistor that is inline
# with the 50K amplifier drain voltage source which is
# used to infer the 50K amplifier drain current. The
# default value of 10 Ohm is the standard value in the BOM
# for cryostat card revision C02 (PC-248-103-02-C02). The
# resistor on that revision of the cryostat card is R54.
Optional('50K_amp_Vd_series_resistor', default=10.0):
And(float, lambda f: f > 0),
# 50K amplifier gate voltage, in volts.
"LNA_Vg":
Use(float),
# Which RTM DAC is wired to the gate of the 50K amplifier.
"dac_num_50k":
And(int, lambda n: 1 <= n <= 32),
# Conversion from bits (the digital value the RTM DAC is set to)
# to volts for the 50K amplifier gate. Units are volts/bit. An
# important dependency is the voltage division on the cryostat
# card, which can be different from cryostat card to cryostat card
"bit_to_V_50k":
And(Use(float), lambda f: f > 0),
# Software limit on the minimum gate voltage that can be set for the 4K amplifier.
"hemt_gate_min_voltage":
Use(float),
# Software limit on the maximum gate voltage that can be set for the 4K amplifier.
"hemt_gate_max_voltage":
Use(float)
}
#### Done specifiying amplifier
#### Start specifying tune parameter schema
schema_dict['tune_band'] = {
'fraction_full_scale': And(Use(float), lambda f: 0 < f <= 1.),
'reset_rate_khz': And(Use(float), lambda f: 0 <= f <= 100),
Optional('default_tune', default=None): And(str, os.path.isfile)
}
## Add tuning params that must be specified per band.
per_band_tuning_params = [
('lms_freq', And(Use(float), lambda f: f > 0)),
('delta_freq', And(Use(float), lambda f: f > 0)),
('feedback_start_frac', And(Use(float), lambda f: 0 <= f <= 1)),
('feedback_end_frac', And(Use(float), lambda f: 0 <= f <= 1)),
('gradient_descent_gain', And(Use(float), lambda f: f > 0)),
('gradient_descent_averages', And(Use(int), lambda n: n > 0)),
('gradient_descent_converge_hz', And(Use(float), lambda f: f > 0)),
('gradient_descent_momentum', And(Use(int), lambda n: n >= 0)),
('gradient_descent_step_hz', And(Use(float), lambda f: f > 0)),
('gradient_descent_beta', And(Use(float), lambda f: 0 <= f <= 1)),
('eta_scan_averages', And(Use(int), lambda n: n > 0)),
('eta_scan_del_f', And(Use(int), lambda n: n > 0)),
]
for band in bands:
for (param, value) in per_band_tuning_params:
if band == bands[0]:
schema_dict['tune_band'][param] = {}
schema_dict['tune_band'][param][str(band)] = value
## Done adding tuning params that must be specified per band.
#### Done specifying tune parameter schema
#### Start specifying bad mask
schema_dict[Optional('bad_mask', default={})] = {
# Why are these indexed by integers that are also strings?
# I don't think the keys here are used at all.
Optional(str):
And([Use(float)], list, lambda l: len(l) == 2 and l[0] < l[1] and
all(4000 <= ll <= 8000 for ll in l))
}
#### Done specifying bad mask
#### Start specifying TES-related
# TES shunt resistance
schema_dict["R_sh"] = And(Use(float), lambda f: f > 0)
# Round-trip resistance on TES bias lines, in low current mode.
# Includes the resistance on the cryostat cards, and cable resistance.
schema_dict["bias_line_resistance"] = And(Use(float), lambda f: f > 0)
# Ratio between the current per DAC unit in high current mode to the
# current in low current mode. Constained to be greater than or equal
# to 1 since otherwise what does high current mode EVEN MEAN.
schema_dict["high_low_current_ratio"] = And(Use(float),
lambda f: f >= 1)
# If 1, TES biasing will *always* be in high current mode.
schema_dict[Optional('high_current_mode_bool',
default=0)] = And(int, lambda n: n in (0, 1))
# All SMuRF bias groups with TESs connected.
schema_dict["all_bias_groups"] = And([int], list,
lambda l: all(0 <= ll < 16
for ll in l))
#### Done specifying TES-related
#### Start specifying flux ramp-related
schema_dict["flux_ramp"] = {
# 20 bits for the C0 RTMs, 32 bits for the C1 RTMs.
"num_flux_ramp_counter_bits": And(int, lambda n: n in (20, 32))
}
#### Done specifying flux-ramp related
#### Start specifying constants schema
# If all of a schema dictionary's keys are optional, must specify
# them both in the schema key for that dictionary, and in the
# schema for that dictionary.
constants_default_dict = {'pA_per_phi0': 9.e6}
cdd_key = Optional("constant", default=constants_default_dict)
schema_dict[cdd_key] = {}
# Assumes all constants default values are floats
for key, value in constants_default_dict.items():
schema_dict[cdd_key][Optional(key, default=value)] = Use(float)
#### Done specifying constants schema
#### Start thermal schema
# OT protection for ultrascale FPGA, in degrees C. If None,
# then pysmurf doesn't try to engage OT protection. For
# unknown reasons, enabling OT protection in the ELMA crate
# we've been using for testing on campus at Stanford takes
# down the carrier after the third command in the enable
# sequence (set_ot_threshold_disable(0)), but it works in the
# RF lab at SLAC where they've been testing with an ASIS
# crate. Shawn has yet to have this work for him. Newer fw
# versions will have OT protection enabled in the fw.
schema_dict[Optional('ultrascale_temperature_limit_degC',
default=None)] = And(Use(float),
lambda f: 0 <= f <= 99)
#### Done specifying thermal schema
#### Start specifying timing-related schema
schema_dict["timing"] = {
# "ext_ref" : internal oscillator locked to an external
# front-panel reference, or unlocked if there is no front
# panel reference. (LmkReg_0x0147 : 0x1A). Also sets
# flux_ramp_start_mode=0
# "backplane" : takes timing from timing master through
# backplane. Also sets flux_ramp_start_mode=1.
"timing_reference": And(str, lambda s: s in
('ext_ref', 'backplane'))
}
#### Done specifying timing-related schema
#### Start specifying smurf2mce
# System should be smart enough to determine fs on the fly.
schema_dict["fs"] = And(Use(float), lambda f: f > 0)
def user_has_write_access(dirpath):
return os.access(dirpath, os.W_OK)
def dir_exists_with_write_access(file_path):
filedir_path = os.path.dirname(file_path)
if (not os.path.isdir(filedir_path)
or not user_has_write_access(filedir_path)):
return False
return True
#### Start specifying directories
schema_dict[Optional("default_data_dir",
default="/data/smurf_data")] = And(
str, os.path.isdir, user_has_write_access)
schema_dict[Optional("smurf_cmd_dir",
default="/data/smurf_data/smurf_cmd")] = And(
str, os.path.isdir, user_has_write_access)
schema_dict[Optional("tune_dir",
default="/data/smurf_data/tune")] = And(
str, os.path.isdir, user_has_write_access)
schema_dict[Optional("status_dir",
default="/data/smurf_data/status")] = And(
str, os.path.isdir, user_has_write_access)
#### Done specifying directories
##### Done building validation schema
###################################################
###################################################
# Validate the full config
schema = Schema(schema_dict, ignore_extra_keys=True)
validated_config = schema.validate(loaded_config)
###################################################
# Higher level/composite validation, if schema validation
# succeeds
# Check that no DAC has been assigned to multiple TES bias groups
bias_group_to_pair = validated_config['bias_group_to_pair']
tes_bias_group_dacs = np.ndarray.flatten(
np.array([bg2p[1] for bg2p in bias_group_to_pair.items()]))
assert (len(
np.unique(tes_bias_group_dacs)) == len(tes_bias_group_dacs)), (
'Configuration failed - DACs may not be ' +
'assigned to multiple TES bias groups.')
# Check that the DAC specified as the 50K gate driver
# isn't also defined as one of the DACs in a TES bias group
# pair.
dac_num_50k = validated_config['amplifier']['dac_num_50k']
# Taking the first element works because we already required
# that no DAC show up in more than one TES bias group
# definition.
if dac_num_50k in tes_bias_group_dacs:
# which TES bias group is defined as using the requested
# DAC for biasing the 50K amplifier?
bias_group = int([
bg2p[0] for bg2p in bias_group_to_pair.items()
if dac_num_50k in bg2p[1]
][0])
assert False, 'Configuration failed - DAC requested ' + \
f'for driving 50K amplifier gate, {dac_num_50k}, is ' + \
f'also assigned to TES bias group {bias_group}.'
##### Done with higher level/composite validation.
###################################################
# Splice in the sorted init:bands key before returning
validated_config['init']['bands'] = bands
return validated_config
class Stage(object):
STAGE_FILE = "Dvcfile"
STAGE_FILE_SUFFIX = ".dvc"
PARAM_MD5 = "md5"
PARAM_CMD = "cmd"
PARAM_WDIR = "wdir"
PARAM_DEPS = "deps"
PARAM_OUTS = "outs"
PARAM_LOCKED = "locked"
PARAM_META = "meta"
PARAM_ALWAYS_CHANGED = "always_changed"
SCHEMA = {
Optional(PARAM_MD5): Or(str, None),
Optional(PARAM_CMD): Or(str, None),
Optional(PARAM_WDIR): Or(str, None),
Optional(PARAM_DEPS): Or(And(list, Schema([dependency.SCHEMA])), None),
Optional(PARAM_OUTS): Or(And(list, Schema([output.SCHEMA])), None),
Optional(PARAM_LOCKED): bool,
Optional(PARAM_META): object,
Optional(PARAM_ALWAYS_CHANGED): bool,
}
TAG_REGEX = r"^(?P<path>.*)@(?P<tag>[^\\/@:]*)$"
def __init__(
self,
repo,
path=None,
cmd=None,
wdir=os.curdir,
deps=None,
outs=None,
md5=None,
locked=False,
tag=None,
state=None,
always_changed=False,
):
if deps is None:
deps = []
if outs is None:
outs = []
self.repo = repo
self.path = path
self.cmd = cmd
self.wdir = wdir
self.outs = outs
self.deps = deps
self.md5 = md5
self.locked = locked
self.tag = tag
self.always_changed = always_changed
self._state = state or {}
def __repr__(self):
return "Stage: '{path}'".format(
path=self.relpath if self.path else "No path")
@property
def relpath(self):
return relpath(self.path)
@property
def is_data_source(self):
"""Whether the DVC-file was created with `dvc add` or `dvc import`"""
return self.cmd is None
@staticmethod
def is_valid_filename(path):
return (
# path.endswith doesn't work for encoded unicode filenames on
# Python 2 and since Stage.STAGE_FILE_SUFFIX is ascii then it is
# not needed to decode the path from py2's str
path[-len(Stage.STAGE_FILE_SUFFIX):] == Stage.STAGE_FILE_SUFFIX
or os.path.basename(path) == Stage.STAGE_FILE)
@staticmethod
def is_stage_file(path):
return os.path.isfile(path) and Stage.is_valid_filename(path)
def changed_md5(self):
return self.md5 != self._compute_md5()
@property
def is_callback(self):
"""
A callback stage is always considered as changed,
so it runs on every `dvc repro` call.
"""
return not self.is_data_source and len(self.deps) == 0
@property
def is_import(self):
"""Whether the DVC-file was created with `dvc import`."""
return not self.cmd and len(self.deps) == 1 and len(self.outs) == 1
@property
def is_repo_import(self):
if not self.is_import:
return False
return isinstance(self.deps[0], dependency.DependencyREPO)
def _changed_deps(self):
if self.locked:
return False
if self.is_callback:
logger.warning(
"DVC-file '{fname}' is a 'callback' stage "
"(has a command and no dependencies) and thus always "
"considered as changed.".format(fname=self.relpath))
return True
if self.always_changed:
return True
for dep in self.deps:
status = dep.status()
if status:
logger.warning(
"Dependency '{dep}' of '{stage}' changed because it is "
"'{status}'.".format(dep=dep,
stage=self.relpath,
status=status[str(dep)]))
return True
return False
def _changed_outs(self):
for out in self.outs:
status = out.status()
if status:
logger.warning(
"Output '{out}' of '{stage}' changed because it is "
"'{status}'".format(out=out,
stage=self.relpath,
status=status[str(out)]))
return True
return False
def _changed_md5(self):
if self.changed_md5():
logger.warning("DVC-file '{}' changed.".format(self.relpath))
return True
return False
def changed(self):
# Short-circuit order: stage md5 is fast, deps are expected to change
ret = (self._changed_md5() or self._changed_deps()
or self._changed_outs())
if ret:
logger.warning("Stage '{}' changed.".format(self.relpath))
else:
logger.info("Stage '{}' didn't change.".format(self.relpath))
return ret
def remove_outs(self, ignore_remove=False, force=False):
"""Used mainly for `dvc remove --outs` and :func:`Stage.reproduce`."""
for out in self.outs:
if out.persist and not force:
out.unprotect()
else:
logger.debug("Removing output '{out}' of '{stage}'.".format(
out=out, stage=self.relpath))
out.remove(ignore_remove=ignore_remove)
def unprotect_outs(self):
for out in self.outs:
out.unprotect()
def remove(self, force=False, remove_outs=True):
if remove_outs:
self.remove_outs(ignore_remove=True, force=force)
else:
self.unprotect_outs()
os.unlink(self.path)
def reproduce(self, interactive=False, **kwargs):
if not kwargs.get("force", False) and not self.changed():
return None
msg = ("Going to reproduce '{stage}'. "
"Are you sure you want to continue?".format(stage=self.relpath))
if interactive and not prompt.confirm(msg):
raise DvcException("reproduction aborted by the user")
self.run(**kwargs)
logger.debug("'{stage}' was reproduced".format(stage=self.relpath))
return self
def update(self):
if not self.is_repo_import and not self.is_import:
raise StageUpdateError(self.relpath)
self.deps[0].update()
locked = self.locked
self.locked = False
try:
self.reproduce()
finally:
self.locked = locked
@staticmethod
def validate(d, fname=None):
from dvc.utils import convert_to_unicode
try:
Schema(Stage.SCHEMA).validate(convert_to_unicode(d))
except SchemaError as exc:
raise StageFileFormatError(fname, exc)
@classmethod
def _stage_fname(cls, outs, add):
if not outs:
return cls.STAGE_FILE
out = outs[0]
fname = out.path_info.name + cls.STAGE_FILE_SUFFIX
if (add and out.is_in_repo
and not contains_symlink_up_to(out.fspath, out.repo.root_dir)):
fname = out.path_info.with_name(fname).fspath
return fname
@staticmethod
def _check_stage_path(repo, path):
assert repo is not None
real_path = os.path.realpath(path)
if not os.path.exists(real_path):
raise StagePathNotFoundError(path)
if not os.path.isdir(real_path):
raise StagePathNotDirectoryError(path)
proj_dir = os.path.realpath(repo.root_dir) + os.path.sep
if not (real_path + os.path.sep).startswith(proj_dir):
raise StagePathOutsideError(path)
@property
def is_cached(self):
"""
Checks if this stage has been already ran and stored
"""
from dvc.remote.local import RemoteLOCAL
from dvc.remote.s3 import RemoteS3
old = Stage.load(self.repo, self.path)
if old._changed_outs():
return False
# NOTE: need to save checksums for deps in order to compare them
# with what is written in the old stage.
for dep in self.deps:
dep.save()
old_d = old.dumpd()
new_d = self.dumpd()
# NOTE: need to remove checksums from old dict in order to compare
# it to the new one, since the new one doesn't have checksums yet.
old_d.pop(self.PARAM_MD5, None)
new_d.pop(self.PARAM_MD5, None)
outs = old_d.get(self.PARAM_OUTS, [])
for out in outs:
out.pop(RemoteLOCAL.PARAM_CHECKSUM, None)
out.pop(RemoteS3.PARAM_CHECKSUM, None)
if old_d != new_d:
return False
# NOTE: committing to prevent potential data duplication. For example
#
# $ dvc config cache.type hardlink
# $ echo foo > foo
# $ dvc add foo
# $ rm -f foo
# $ echo foo > foo
# $ dvc add foo # should replace foo with a link to cache
#
old.commit()
return True
@staticmethod
def create(repo, **kwargs):
wdir = kwargs.get("wdir", None)
cwd = kwargs.get("cwd", None)
fname = kwargs.get("fname", None)
add = kwargs.get("add", False)
# Backward compatibility for `cwd` option
if wdir is None and cwd is not None:
if fname is not None and os.path.basename(fname) != fname:
raise StageFileBadNameError(
"DVC-file name '{fname}' may not contain subdirectories"
" if '-c|--cwd' (deprecated) is specified. Use '-w|--wdir'"
" along with '-f' to specify DVC-file path with working"
" directory.".format(fname=fname))
wdir = cwd
elif wdir is None:
wdir = os.curdir
stage = Stage(
repo=repo,
wdir=wdir,
cmd=kwargs.get("cmd", None),
locked=kwargs.get("locked", False),
always_changed=kwargs.get("always_changed", False),
)
Stage._fill_stage_outputs(stage, **kwargs)
stage.deps = dependency.loads_from(stage,
kwargs.get("deps", []),
erepo=kwargs.get("erepo", None))
stage._check_circular_dependency()
stage._check_duplicated_arguments()
if not fname:
fname = Stage._stage_fname(stage.outs, add)
stage._check_dvc_filename(fname)
# Autodetecting wdir for add, we need to create outs first to do that,
# so we start with wdir = . and remap out paths later.
if add and kwargs.get("wdir") is None and cwd is None:
wdir = os.path.dirname(fname)
for out in chain(stage.outs, stage.deps):
if out.is_in_repo:
out.def_path = relpath(out.path_info, wdir)
wdir = os.path.abspath(wdir)
if cwd is not None:
path = os.path.join(wdir, fname)
else:
path = os.path.abspath(fname)
Stage._check_stage_path(repo, wdir)
Stage._check_stage_path(repo, os.path.dirname(path))
stage.wdir = wdir
stage.path = path
ignore_build_cache = kwargs.get("ignore_build_cache", False)
# NOTE: remove outs before we check build cache
if kwargs.get("remove_outs", False):
logger.warning("--remove-outs is deprecated."
" It is now the default behavior,"
" so there's no need to use this option anymore.")
stage.remove_outs(ignore_remove=False)
logger.warning("Build cache is ignored when using --remove-outs.")
ignore_build_cache = True
if os.path.exists(path) and any(out.persist for out in stage.outs):
logger.warning("Build cache is ignored when persisting outputs.")
ignore_build_cache = True
if os.path.exists(path):
if (not ignore_build_cache and stage.is_cached
and not stage.is_callback and not stage.always_changed):
logger.info("Stage is cached, skipping.")
return None
msg = ("'{}' already exists. Do you wish to run the command and "
"overwrite it?".format(stage.relpath))
if not kwargs.get("overwrite", True) and not prompt.confirm(msg):
raise StageFileAlreadyExistsError(stage.relpath)
os.unlink(path)
return stage
@staticmethod
def _fill_stage_outputs(stage, **kwargs):
stage.outs = output.loads_from(stage,
kwargs.get("outs", []),
use_cache=True)
stage.outs += output.loads_from(stage,
kwargs.get("metrics", []),
use_cache=True,
metric=True)
stage.outs += output.loads_from(stage,
kwargs.get("outs_persist", []),
use_cache=True,
persist=True)
stage.outs += output.loads_from(stage,
kwargs.get("outs_no_cache", []),
use_cache=False)
stage.outs += output.loads_from(
stage,
kwargs.get("metrics_no_cache", []),
use_cache=False,
metric=True,
)
stage.outs += output.loads_from(
stage,
kwargs.get("outs_persist_no_cache", []),
use_cache=False,
persist=True,
)
@staticmethod
def _check_dvc_filename(fname):
if not Stage.is_valid_filename(fname):
raise StageFileBadNameError(
"bad DVC-file name '{}'. DVC-files should be named"
" 'Dvcfile' or have a '.dvc' suffix (e.g. '{}.dvc').".format(
relpath(fname), os.path.basename(fname)))
@staticmethod
def _check_file_exists(repo, fname):
if not repo.tree.exists(fname):
raise StageFileDoesNotExistError(fname)
@staticmethod
def _check_isfile(repo, fname):
if not repo.tree.isfile(fname):
raise StageFileIsNotDvcFileError(fname)
@classmethod
def _get_path_tag(cls, s):
regex = re.compile(cls.TAG_REGEX)
match = regex.match(s)
if not match:
return s, None
return match.group("path"), match.group("tag")
@staticmethod
def load(repo, fname):
fname, tag = Stage._get_path_tag(fname)
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a DVC-file
# 3. path doesn't represent a regular file
Stage._check_file_exists(repo, fname)
Stage._check_dvc_filename(fname)
Stage._check_isfile(repo, fname)
with repo.tree.open(fname) as fd:
d = load_stage_fd(fd, fname)
# Making a deepcopy since the original structure
# looses keys in deps and outs load
state = copy.deepcopy(d)
Stage.validate(d, fname=relpath(fname))
path = os.path.abspath(fname)
stage = Stage(
repo=repo,
path=path,
wdir=os.path.abspath(
os.path.join(os.path.dirname(path),
d.get(Stage.PARAM_WDIR, "."))),
cmd=d.get(Stage.PARAM_CMD),
md5=d.get(Stage.PARAM_MD5),
locked=d.get(Stage.PARAM_LOCKED, False),
tag=tag,
always_changed=d.get(Stage.PARAM_ALWAYS_CHANGED, False),
state=state,
)
stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
return stage
def dumpd(self):
rel_wdir = relpath(self.wdir, os.path.dirname(self.path))
wdir = pathlib.PurePath(rel_wdir).as_posix()
wdir = wdir if wdir != "." else None
return {
key: value
for key, value in {
Stage.PARAM_MD5: self.md5,
Stage.PARAM_CMD: self.cmd,
Stage.PARAM_WDIR: wdir,
Stage.PARAM_LOCKED: self.locked,
Stage.PARAM_DEPS: [d.dumpd() for d in self.deps],
Stage.PARAM_OUTS: [o.dumpd() for o in self.outs],
Stage.PARAM_META: self._state.get("meta"),
Stage.PARAM_ALWAYS_CHANGED: self.always_changed,
}.items() if value
}
def dump(self):
fname = self.path
self._check_dvc_filename(fname)
logger.info(
"Saving information to '{file}'.".format(file=relpath(fname)))
d = self.dumpd()
apply_diff(d, self._state)
dump_stage_file(fname, self._state)
self.repo.scm.track_file(relpath(fname))
def _compute_md5(self):
from dvc.output.base import OutputBase
d = self.dumpd()
# Remove md5 and meta, these should not affect stage md5
d.pop(self.PARAM_MD5, None)
d.pop(self.PARAM_META, None)
# Ignore the wdir default value. In this case DVC-file w/o
# wdir has the same md5 as a file with the default value specified.
# It's important for backward compatibility with pipelines that
# didn't have WDIR in their DVC-files.
if d.get(self.PARAM_WDIR) == ".":
del d[self.PARAM_WDIR]
# NOTE: excluding parameters that don't affect the state of the
# pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
# it has changed, we might not have that output in our cache.
m = dict_md5(
d,
exclude=[
self.PARAM_LOCKED,
OutputBase.PARAM_METRIC,
OutputBase.PARAM_TAGS,
OutputBase.PARAM_PERSIST,
],
)
logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m))
return m
def save(self):
for dep in self.deps:
dep.save()
for out in self.outs:
out.save()
self.md5 = self._compute_md5()
@staticmethod
def _changed_entries(entries):
return [
str(entry) for entry in entries
if entry.checksum and entry.changed_checksum()
]
def check_can_commit(self, force):
changed_deps = self._changed_entries(self.deps)
changed_outs = self._changed_entries(self.outs)
if changed_deps or changed_outs or self.changed_md5():
msg = ("dependencies {}".format(changed_deps)
if changed_deps else "")
msg += " and " if (changed_deps and changed_outs) else ""
msg += "outputs {}".format(changed_outs) if changed_outs else ""
msg += "md5" if not (changed_deps or changed_outs) else ""
msg += " of '{}' changed. Are you sure you commit it?".format(
self.relpath)
if not force and not prompt.confirm(msg):
raise StageCommitError(
"unable to commit changed '{}'. Use `-f|--force` to "
"force.`".format(self.relpath))
self.save()
def commit(self):
for out in self.outs:
out.commit()
def _check_missing_deps(self):
missing = [dep for dep in self.deps if not dep.exists]
if any(missing):
raise MissingDep(missing)
@staticmethod
def _warn_if_fish(executable): # pragma: no cover
if (executable is None
or os.path.basename(os.path.realpath(executable)) != "fish"):
return
logger.warning(
"DVC detected that you are using fish as your default "
"shell. Be aware that it might cause problems by overwriting "
"your current environment variables with values defined "
"in '.fishrc', which might affect your command. See "
"https://github.com/iterative/dvc/issues/1307. ")
def _check_circular_dependency(self):
from dvc.exceptions import CircularDependencyError
circular_dependencies = set(d.path_info for d in self.deps) & set(
o.path_info for o in self.outs)
if circular_dependencies:
raise CircularDependencyError(str(circular_dependencies.pop()))
def _check_duplicated_arguments(self):
from dvc.exceptions import ArgumentDuplicationError
from collections import Counter
path_counts = Counter(edge.path_info for edge in self.deps + self.outs)
for path, occurrence in path_counts.items():
if occurrence > 1:
raise ArgumentDuplicationError(str(path))
def _run(self):
self._check_missing_deps()
executable = os.getenv("SHELL") if os.name != "nt" else None
self._warn_if_fish(executable)
main_thread = isinstance(threading.current_thread(),
threading._MainThread)
old_handler = None
p = None
try:
p = subprocess.Popen(
self.cmd,
cwd=self.wdir,
shell=True,
env=fix_env(os.environ),
executable=executable,
close_fds=True,
)
if main_thread:
old_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
p.communicate()
finally:
if old_handler:
signal.signal(signal.SIGINT, old_handler)
if (p is None) or (p.returncode != 0):
raise StageCmdFailedError(self)
def run(self, dry=False, no_commit=False, force=False):
if (self.cmd or self.is_import) and not self.locked and not dry:
self.remove_outs(ignore_remove=False, force=False)
if self.locked:
logger.info("Verifying outputs in locked stage '{stage}'".format(
stage=self.relpath))
if not dry:
self.check_missing_outputs()
elif self.is_import:
logger.info("Importing '{dep}' -> '{out}'".format(
dep=self.deps[0], out=self.outs[0]))
if not dry:
if not force and self._already_cached():
self.outs[0].checkout()
else:
self.deps[0].download(self.outs[0])
elif self.is_data_source:
msg = "Verifying data sources in '{}'".format(self.relpath)
logger.info(msg)
if not dry:
self.check_missing_outputs()
else:
logger.info("Running command:\n\t{}".format(self.cmd))
if not dry:
if (not force and not self.is_callback
and not self.always_changed
and self._already_cached()):
self.checkout()
else:
self._run()
if not dry:
self.save()
if not no_commit:
self.commit()
def check_missing_outputs(self):
paths = [str(out) for out in self.outs if not out.exists]
if paths:
raise MissingDataSource(paths)
def checkout(self, force=False, progress_callback=None):
for out in self.outs:
out.checkout(force=force,
tag=self.tag,
progress_callback=progress_callback)
@staticmethod
def _status(entries):
ret = {}
for entry in entries:
ret.update(entry.status())
return ret
def status(self):
ret = []
if not self.locked:
deps_status = self._status(self.deps)
if deps_status:
ret.append({"changed deps": deps_status})
outs_status = self._status(self.outs)
if outs_status:
ret.append({"changed outs": outs_status})
if self.changed_md5():
ret.append("changed checksum")
if self.is_callback or self.always_changed:
ret.append("always changed")
if ret:
return {self.relpath: ret}
return {}
def _already_cached(self):
return (not self.changed_md5()
and all(not dep.changed() for dep in self.deps)
and all(not out.changed_cache() if out.
use_cache else not out.changed() for out in self.outs))
def get_all_files_number(self):
return sum(out.get_files_number() for out in self.outs)
'user': assignee.user.username,
'reason': assignee.reason,
}
def is_int(s):
try:
int(s)
return True
except ValueError:
return False
machine_schema = Schema({
Optional('netboot_enabled'): bool,
Optional('subarch'): Or(None, And(str, lambda s: validators.length(s, min=0, max=256))),
Optional('preseed_id'): Or(None, Use(int)),
Optional('kernel_id'): Or(None, Use(int)),
Optional('kernel_opts'): And(str, lambda s: validators.length(s, min=0, max=1024)),
Optional('initrd_id'): Or(None, Use(int)),
}, ignore_extra_keys=True)
machine_power_schema = Schema({
'state': And(str, lambda s: s in ('on', 'off', 'reboot', 'pxe_reboot', 'bios_reboot', 'disk_reboot')),
})
machine_state_post_schema = Schema({
'state': And(str, lambda s: s in ('provision')),
})
def test_json_schema_and_simplify():
s = And('test')
assert s.json_schema() == {'enum': ['test']}
from schema import Schema, And, Use
from engine.helpers.const.service_characteristics import TYPE, REGIONS, DEPLOYMENT_TIME, LEASING_PERIOD
upload_service_schema = Schema({
'productName':
Use(str),
'Service':
Use(str),
'logo':
Use(str),
'fileName':
Use(str),
'description':
Use(str),
'region': [And(str, Use(str.upper), lambda s: s in REGIONS)],
'serviceType': [And(str, Use(str.upper), lambda s: s in TYPE)],
'attackType': [And(Use(str.upper))],
'deploymentTime':
And(str, Use(str.upper), lambda s: s in DEPLOYMENT_TIME),
'leasingPeriod':
And(str, Use(str.upper), lambda s: s in LEASING_PERIOD),
'price':
And(Use(int), lambda n: n > 0),
'txHash':
Use(str),
'serviceHash':
Use(str)
})
import json
from requests.exceptions import ConnectionError
from schema import Schema, And, Use, Optional
import requests
from os.path import expanduser
import os
vul_schema = Schema(
{
'name': str,
'tool': str,
'description': str,
'project': str,
'target': str,
'scan': str,
Optional('cwe'): And(Use(int)),
Optional('observation'): str,
Optional('severity'): And(Use(int), lambda n: 0 <= n <= 3),
Optional('remediation'): str,
Optional('evidences'): list
},
ignore_extra_keys=False)
evidence_schema = Schema(
{
'name': str,
'url': str,
'vulnId': str,
Optional('param'): str,
Optional('log'): str,
Optional('attack'): str,
return result
else:
return tuple([int(s) for s in data])
def percent_to_float(data):
if isinstance(data, str) and re.match(r'-?\d+(\.\d+)?%', data):
data = float(data.strip('%')) / 100
if isinstance(data, int):
data = float(data)
else:
assert isinstance(data, float), 'This field should be float, int or percent string'
return data
policy_schema = Schema({
Optional('weights', default=None): list,
Optional('method', default=None): And(str, lambda s: s in ["per_channel", "per_tensor"]),
Optional('init_sparsity', default=0): And(float, lambda s: s < 1.0 and s >= 0.0),
Hook('target_sparsity', handler=_valid_prune_sparsity): object,
Optional('target_sparsity', default=0.5): float,
Optional("start_epoch", default=0): int,
Hook('end_epoch', handler=_valid_prune_epoch): object,
Optional('end_epoch', default=4): int
})
ops_schema = Schema({
Optional('weight', default=None): {
Optional('granularity', default=None): And(
list,
lambda s: all(i in ['per_channel', 'per_tensor'] for i in s)),
Optional('scheme', default=None): And(
list,
class Page(object):
"Banana banana"
meta_schema = {
Optional('title'): And(str, len),
Optional('symbols'): Schema([And(str, len)]),
Optional('short-description'): And(str, len),
Optional('render-subpages'): bool,
Optional('auto-sort'): bool,
Optional('full-width'): bool
}
# pylint: disable=too-many-arguments
def __init__(self,
source_file,
ast,
output_path,
project_name,
meta=None,
raw_contents=None):
"Banana banana"
assert source_file
basename = os.path.basename(source_file)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-',
os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.ast = ast
self.extension_name = None
self.source_file = source_file
self.raw_contents = raw_contents
self.comment = None
self.generated = False
self.pre_sorted = False
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.typed_symbols = {}
self.is_stale = True
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.project_name = project_name
self.cached_paths = OrderedSet()
meta = meta or {}
try:
self.meta = Schema(Page.meta_schema).validate(meta)
except SchemaError as _:
warn('invalid-page-metadata',
'%s: Invalid metadata: \n%s' % (self.source_file, str(_)))
self.meta = meta
self.symbol_names = OrderedSet(meta.get('symbols') or [])
self.short_description = meta.get('short-description')
self.render_subpages = meta.get('render-subpages', True)
self.title = None
self.__discover_title(meta)
self.link = Link(pagename, self.title or name, ref)
def __getstate__(self):
return {
'ast': None,
'build_path': None,
'title': self.title,
'raw_contents': self.raw_contents,
'short_description': self.short_description,
'extension_name': self.extension_name,
'link': self.link,
'meta': self.meta,
'source_file': self.source_file,
'comment': self.comment,
'generated': self.generated,
'is_stale': False,
'formatted_contents': None,
'detailed_description': None,
'output_attrs': None,
'symbols': [],
'typed_symbols': {},
'subpages': self.subpages,
'symbol_names': self.symbol_names,
'project_name': self.project_name,
'pre_sorted': self.pre_sorted,
'cached_paths': self.cached_paths,
'render_subpages': self.render_subpages
}
def resolve_symbols(self, tree, database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `database`, and added to lists of actual symbols, sorted
by symbol class.
"""
typed_symbols_list = namedtuple('TypedSymbolsList',
['name', 'symbols'])
for subclass in all_subclasses(Symbol):
self.typed_symbols[subclass] = typed_symbols_list(
subclass.get_plural_name(), [])
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, tree, link_resolver,
database)
if tree.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver, page_path)
self.symbol_names.add(sym.unique_name)
for sym_type in [
ClassSymbol, AliasSymbol, InterfaceSymbol, StructSymbol
]:
syms = self.typed_symbols[sym_type].symbols
if not syms:
continue
if self.title is None:
self.title = syms[0].display_name
if self.comment is None:
self.comment = Comment(name=self.source_file)
self.comment.short_description = syms[
0].comment.short_description
self.comment.title = syms[0].comment.title
break
# pylint: disable=no-self-use
def __fetch_comment(self, sym, database):
old_comment = sym.comment
new_comment = database.get_comment(sym.unique_name)
sym.comment = Comment(sym.unique_name)
if new_comment:
sym.comment = new_comment
elif old_comment:
if old_comment.filename not in (ChangeTracker.all_stale_files |
ChangeTracker.all_unlisted_files):
sym.comment = old_comment
def __format_page_comment(self, formatter, link_resolver):
if not self.comment:
return
if self.comment.short_description:
self.short_description = formatter.format_comment(
self.comment.short_description, link_resolver).strip()
if self.short_description.startswith('<p>'):
self.short_description = self.short_description[3:-4]
if self.comment.title:
self.title = formatter.format_comment(self.comment.title,
link_resolver).strip()
if self.title.startswith('<p>'):
self.title = self.title[3:-4]
if self.title:
self.formatted_contents += '<h1>%s</h1>' % self.title
self.formatted_contents += formatter.format_comment(
self.comment, link_resolver)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.source_file:
title = os.path.splitext(self.source_file)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
self.build_path = os.path.join(formatter.get_output_folder(self),
self.link.ref)
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(diag.code,
message=diag.message,
filename=self.source_file)
self.formatted_contents += out
if not self.formatted_contents:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.cache_page(self)
# pylint: disable=no-self-use
def get_title(self):
"""
Banana banana
"""
return self.title or 'unnamed'
def __discover_title(self, meta):
if meta is not None and 'title' in meta:
self.title = meta['title']
elif self.ast:
self.title = cmark.title_from_ast(self.ast)
def __format_symbols(self, formatter, link_resolver):
for symbol in self.symbols:
if symbol is None:
continue
debug(
'Formatting symbol %s in page %s' %
(symbol.unique_name, self.source_file), 'formatting')
symbol.skip = not formatter.format_symbol(symbol, link_resolver)
def __query_extra_symbols(self, sym, all_syms, tree, link_resolver,
database):
if sym:
self.__fetch_comment(sym, database)
new_symbols = sum(tree.resolving_symbol_signal(self, sym), [])
all_syms.add(sym)
for symbol in new_symbols:
self.__query_extra_symbols(symbol, all_syms, tree,
link_resolver, database)
def __resolve_symbol(self, symbol, link_resolver, page_path):
symbol.resolve_links(link_resolver)
symbol.link.ref = "%s#%s" % (page_path, symbol.unique_name)
for link in symbol.get_extra_links():
link.ref = "%s#%s" % (page_path, link.id_)
tsl = self.typed_symbols.get(type(symbol))
if tsl:
tsl.symbols.append(symbol)
self.symbols.append(symbol)
debug(
'Resolved symbol %s to page %s' %
(symbol.display_name, self.link.ref), 'resolution')
class Page:
"Banana banana"
meta_schema = {
Optional('title'): And(str, len),
Optional('short-description'): And(str),
Optional('description'): And(str),
Optional('render-subpages'): bool,
Optional('auto-sort'): bool,
Optional('full-width'): bool,
Optional('see-also'): And(str, len),
Optional('extra'): Schema({str: object}),
Optional('thumbnail'): And(str, len),
Optional('include'): And(str, len),
}
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
def __init__(self,
name,
generated,
project_name,
extension_name,
source_file=None,
ast=None,
output_path='',
raw_contents=None,
comment=None,
meta=None,
pre_sorted=False,
symbol_names=None):
assert name
if not generated:
assert source_file is not None
self.name = name
basename = os.path.basename(name)
name = os.path.splitext(basename)[0]
ref = os.path.join(output_path,
re.sub(r'\W+', '-',
os.path.splitext(basename)[0]))
pagename = '%s.html' % ref
self.generated = generated
self.project_name = project_name
self.extension_name = extension_name
self.source_file = source_file
self.ast = ast
self.raw_contents = raw_contents
self.comment = comment
self.pre_sorted = pre_sorted
self.symbol_names = OrderedSet(symbol_names or [])
self.output_attrs = None
self.subpages = OrderedSet()
self.symbols = []
self.private_symbols = []
self.typed_symbols = OrderedDict()
self.by_parent_symbols = OrderedDict()
self.formatted_contents = None
self.detailed_description = None
self.build_path = None
self.cached_paths = OrderedSet()
if comment:
meta = comment.meta
elif meta:
meta = meta
else:
meta = {}
self.meta = {}
for key, value in meta.items():
try:
self.meta.update(
Schema(Page.meta_schema).validate(
{key.replace('_', '-').lower(): value}))
except SchemaError as err:
warn(
'invalid-page-metadata',
'%s: Invalid metadata: \n%s, discarding metadata' %
(self.name, str(err)))
if not self.meta.get('extra'):
self.meta['extra'] = defaultdict()
self.title = self.meta.get(
'title',
cmark.title_from_ast(self.ast) if ast else '')
self.thumbnail = self.meta.get('thumbnail')
self.short_description = self.meta.get('short-description', None)
self.render_subpages = self.meta.get('render-subpages', True)
self.link = Link(pagename, self.title or name, ref)
def __repr__(self):
return "<Page %s>" % self.name
@staticmethod
def __get_empty_typed_symbols():
typed_symbols_list = namedtuple('TypedSymbolsList',
['name', 'symbols'])
empty_typed_symbols = {}
for subclass in all_subclasses(Symbol):
empty_typed_symbols[subclass] = typed_symbols_list(
subclass.get_plural_name(), [])
return empty_typed_symbols
def resolve_symbols(self, tree, database, link_resolver):
"""
When this method is called, the page's symbol names are queried
from `database`, and added to lists of actual symbols, sorted
by symbol class.
"""
self.typed_symbols = self.__get_empty_typed_symbols()
all_syms = OrderedSet()
for sym_name in self.symbol_names:
sym = database.get_symbol(sym_name)
self.__query_extra_symbols(sym, all_syms, tree, link_resolver,
database)
if tree.project.is_toplevel:
page_path = self.link.ref
else:
page_path = self.project_name + '/' + self.link.ref
if self.meta.get("auto-sort", True):
all_syms = sorted(all_syms, key=lambda x: x.unique_name)
for sym in all_syms:
sym.update_children_comments()
self.__resolve_symbol(sym, link_resolver, page_path)
self.symbol_names.add(sym.unique_name)
# Always put symbols with no parent at the end
no_parent_syms = self.by_parent_symbols.pop(None, None)
if no_parent_syms:
self.by_parent_symbols[None] = no_parent_syms
for sym_type in [
ClassSymbol, AliasSymbol, InterfaceSymbol, StructSymbol
]:
syms = self.typed_symbols[sym_type].symbols
if not syms:
continue
if self.title is None:
self.title = syms[0].display_name
if self.comment is None:
self.comment = Comment(name=self.name)
self.comment.short_description = syms[
0].comment.short_description
self.comment.title = syms[0].comment.title
break
# pylint: disable=no-self-use
def __fetch_comment(self, sym, database):
sym.comment = database.get_comment(sym.unique_name) or Comment(
sym.unique_name)
for sym in sym.get_children_symbols():
if isinstance(sym, Symbol):
self.__fetch_comment(sym, database)
def __format_page_comment(self, formatter, link_resolver):
if not self.comment:
return
if self.comment.short_description:
self.short_description = formatter.format_comment(
self.comment.short_description, link_resolver).strip()
if self.short_description.startswith('<p>'):
self.short_description = self.short_description[3:-4]
if self.comment.title:
self.title = formatter.format_comment(self.comment.title,
link_resolver).strip()
if self.title.startswith('<p>'):
self.title = self.title[3:-4]
if self.title:
self.formatted_contents += '<h1 id="%s-page">%s</h1>' % (
id_from_text(self.title), self.title)
self.formatted_contents += formatter.format_comment(
self.comment, link_resolver)
def format(self, formatter, link_resolver, output):
"""
Banana banana
"""
if not self.title and self.name:
title = os.path.splitext(self.name)[0]
self.title = os.path.basename(title).replace('-', ' ')
self.formatted_contents = u''
self.build_path = os.path.join(formatter.get_output_folder(self),
self.link.ref)
if self.ast:
out, diags = cmark.ast_to_html(self.ast, link_resolver)
for diag in diags:
warn(diag.code,
message=diag.message,
filename=self.source_file or self.name)
self.formatted_contents += out
if not self.formatted_contents:
self.__format_page_comment(formatter, link_resolver)
self.output_attrs = defaultdict(lambda: defaultdict(dict))
formatter.prepare_page_attributes(self)
self.__format_symbols(formatter, link_resolver)
self.detailed_description =\
formatter.format_page(self)[0]
if output:
formatter.cache_page(self)
# pylint: disable=no-self-use
def get_title(self):
"""
Banana banana
"""
return self.title or 'unnamed'
def __discover_title(self, meta):
if meta is not None and 'title' in meta:
self.title = meta['title']
elif self.ast:
self.title = cmark.title_from_ast(self.ast)
def __format_symbols(self, formatter, link_resolver):
for symbol in self.symbols:
if symbol is None:
continue
debug(
'Formatting symbol %s in page %s' %
(symbol.unique_name, self.name), 'formatting')
symbol.detailed_description = formatter.format_symbol(
symbol, link_resolver)
def __query_extra_symbols(self, sym, all_syms, tree, link_resolver,
database):
if sym:
self.__fetch_comment(sym, database)
all_syms.add(sym)
def __resolve_symbol(self, symbol, link_resolver, page_path):
symbol.resolve_links(link_resolver)
symbol.link.ref = "%s#%s" % (page_path, symbol.unique_name)
for link in symbol.get_extra_links():
link.ref = "%s#%s" % (page_path, link.id_)
tsl = self.typed_symbols.get(type(symbol))
if tsl:
tsl.symbols.append(symbol)
by_parent_symbols = self.by_parent_symbols.get(symbol.parent_name)
if not by_parent_symbols:
by_parent_symbols = self.__get_empty_typed_symbols()
parent_name = symbol.parent_name
if parent_name is None:
parent_name = 'Others symbols'
self.by_parent_symbols[symbol.parent_name] = by_parent_symbols
by_parent_symbols.get(type(symbol)).symbols.append(symbol)
self.symbols.append(symbol)
debug(
'Resolved symbol %s to page %s' %
(symbol.unique_name, self.link.ref), 'resolution')
Optional("notification_endpoint"): str,
Optional("schedule"): str,
Optional("restart_execution_on_update"): bool,
Optional("pipeline_type", default="default"): Or("default"),
}
AWS_ACCOUNT_ID_REGEX_STR = r"\A[0-9]{12}\Z"
AWS_ACCOUNT_ID_SCHEMA = Schema(
And(
Or(int, str), Use(str),
Regex(
AWS_ACCOUNT_ID_REGEX_STR,
error=(
"The specified account id is incorrect. "
"This typically happens when you specify the account id as a "
"number, while the account id starts with a zero. If this is "
"the case, please wrap the account id in quotes to make it a "
"string. An AWS Account Id is a number of 12 digits, which "
"should start with a zero if the Account Id has a zero at "
"the start too. "
"The number shown to not match the regular expression could "
"be interpreted as an octal number due to the leading zero. "
"Therefore, it might not match the account id as specified "
"in the deployment map."))))
# CodeCommit Source
CODECOMMIT_SOURCE_PROPS = {
"account_id": AWS_ACCOUNT_ID_SCHEMA,
Optional("repository"): str,
Optional("branch"): str,
Optional("poll_for_changes"): bool,
Optional("owner"): str,
def percent_to_float(data):
if isinstance(data, str) and re.match(r'-?\d+(\.\d+)?%', data):
data = float(data.strip('%')) / 100
else:
assert isinstance(
data, float), 'This field should be float or percent string'
return data
policy_schema = Schema({
Optional('weights', default=None):
list,
Optional('method', default=None):
And(str, lambda s: s in ["per_channel", "per_tensor"]),
Optional('init_sparsity', default=0):
And(float, lambda s: s < 1.0 and s >= 0.0),
Hook('target_sparsity', handler=_valid_prune_sparsity):
object,
Optional('target_sparsity', default=0.5):
float,
Optional("start_epoch", default=0):
int,
Hook('end_epoch', handler=_valid_prune_epoch):
object,
Optional('end_epoch', default=4):
int
})
ops_schema = Schema({
__credits__ = ["Costas Tyfoxylos"]
__license__ = '''MIT'''
__maintainer__ = '''Costas Tyfoxylos'''
__email__ = '''<*****@*****.**>'''
__status__ = '''Development''' # "Prototype", "Development", "Production".
def is_valid_regex(value):
"""Validates a regex"""
try:
re.compile(value)
is_valid = True
except re.error:
is_valid = False
return is_valid
NAMING_SCHEMA = Schema([{
'resource':
str,
'regex':
is_valid_regex,
Optional('fields'): [{
'value': str,
'regex': is_valid_regex
}]
}])
POSITIONING_SCHEMA = Schema({And(str, lambda x: x.endswith('.tf')): [str]})
DISASTER_RECOVERY_FILENAME = 'disaster_recovery.tf'
def test_schema_repr(): # what about repr with `error`s?
schema = Schema([Or(None, And(str, Use(float)))])
repr_ = "Schema([Or(None, And(<type 'str'>, Use(<type 'float'>)))])"
# in Python 3 repr contains <class 'str'>, not <type 'str'>
assert repr(schema).replace('class', 'type') == repr_
#
#
#
#
#
import csv
from cytoolz.dicttoolz import assoc
from cytoolz.functoolz import thread_first
from schema import Schema, And, Or
###################################################################
## Schemas
barcodeSchema = Schema({
"barcode": And(str, lambda s: 12 <= len(s) <= 30),
"forward_barcode": And(str, lambda s: 6 <= len(s) <= 15),
"forward_spacer": And(str, lambda s: 0 <= len(s) <= 10),
"forward_primer": And(str, lambda s: 12 <= len(s) <= 35),
"reverse_barcode": And(str, lambda s: 6 <= len(s) <= 15),
"reverse_spacer": And(str, lambda s: 0 <= len(s) <= 10),
"reverse_primer": And(str, lambda s: 12 <= len(s) <= 35)
})
fastadataSchema = Schema({
"forward_id": str,
"forward_desc": str,
"forward_sequence": str,
"reverse_id": str,
"reverse_desc": str,
"reverse_sequence": str,
def decoder(self):
return Or(
And(int, Use(self.type)),
And(str, Use(lambda x: self.type[x.upper()])),
)
from schema import Schema, And, Or, Optional, Use
states = [
"QUEUED", "RECEIVED", "STARTED", "SUCCEEDED", "RETRY", "REVOKED", "FAILED",
"REJECTED"
]
TriggerSchema = Schema({
"enabled":
And(bool),
Optional("states", default=[]): [str],
Optional("envs", default=[]): [str],
Optional("runtime_upper_bound"):
And(Use(float), lambda n: 0.000000000001 <= n <= 1000),
Optional(Or("exclude", "include", only_one=True)): [str],
"type":
Or("slack"),
"slack_wh_url":
And(str, len),
})
ApplicationSchema = Schema({
"app_name":
And(str, lambda e: e.isalpha() and e.islower()),
"app_description":
And(str, len),
Optional("fo_triggers", default=[]): []
})
def encoder(self):
return And(self.type, Or(
And(lambda x: not x.name.startswith('RESERVED_'),
Use(lambda x: x.name.lower())),
Use(lambda x: x.value),
))
def test_schema_repr(): # what about repr with `error`s?
schema = Schema([Or(None, And(str, Use(float)))])
# repr_ = "Schema([Or(None, And(<type 'str'>, Use(<type 'float'>)))])"
repr_ = "Schema(List(Or(Schema(None), And(Schema(<type 'str'>), Use(<type 'float'>)))))"
# in Python 3 repr contains <class 'str'>, not <type 'str'>
assert repr(schema).replace("class", "type") == repr_
def decoder(self):
return Or(
And(int, Use(self.type)),
And(str, Use(lambda x: self.type[re.sub(r'(\d+)\s*mA$', r'MA_\1',
x.upper(), flags=re.I)])),
)
})
MOVE_SCHEMA = Schema({
'type': Use(str),
'id': Use(str),
'quantity': Use(int),
})
NETWORK_SCHEMA = Schema({
'ssid': Use(str),
'password': Use(str),
})
JOB_SCHEMA = Schema({
'job_type':
And(str, Use(str.lower), lambda s: s in ('track', 'learn', 'secure')),
'scenario':
Or(None, Use(str)),
'predicted_mission':
Use(bool),
'recognized_persons':
Or(None, Use(list)),
'non_moving_target':
Or(None, Use(bool)),
Optional('arm_expansion'):
Or(None, Use(bool)),
'ai':
Or(None, Use(str)),
})
FACE_ENCODING_SCHEMA = Schema({
def encoder(self):
return And(self.type, Or(
And(lambda x: not x.name.startswith('RESERVED_'),
Use(lambda x: re.sub(r'ma_(\d+)$', r'\1 mA', x.name.lower()))),
Use(lambda x: x.value),
))
from http import HTTPStatus
from flask import request, jsonify, Blueprint, current_app
from schema import Schema, SchemaError, And
from account_service.domain import commands
from account_service.domain.account import Account
from account_service.domain.errors import CustomerNotFound, AccountNotFound
accounts = Blueprint('accounts', __name__, url_prefix='/accounts/')
POST_ACCOUNT_PAYLOAD_SCHEMA = Schema({'customerId': And(str, len)})
@accounts.route('/<int:account_number>', methods=['GET'])
def get_account(account_number):
account = commands.get_account(
account_number=account_number,
account_repository=current_app.account_repository)
return jsonify(accountNumber=account.formatted_account_number,
accountStatus=account.account_status,
customerId=account.customer_id)
@accounts.route('/<int:account_number>', methods=['PUT'])
def update_account(account_number):
body = request.get_json()
commands.update_account(account_number=account_number,
status=body['status'],
class EepromDescription:
"""EEPROM description"""
# pylint: disable=too-many-instance-attributes
data: Dict
decoder: ClassVar[Schema] = Schema({
Optional('uuid'): And(str, Use(UUID)),
Optional('pid'): int,
Optional('pver'): int,
Optional('vstr'): And(str, Use(str.encode)),
Optional('pstr'): And(str, Use(str.encode)),
Optional('drive'): EepromDriveValue.decoder,
Optional('slew'): EepromSlewValue.decoder,
Optional('hysteresis'): EepromHysteresisValue.decoder,
Optional('back_power'): EepromBackPowerValue.decoder,
Optional('pins'): Or({}, {
Or(int, And(str, Use(int))): {
Optional('function'): EepromPinFunctionValue.decoder,
Optional('pull'): EepromPinPullValue.decoder,
},
}),
})
encoder: ClassVar[Schema] = Schema({
Optional('uuid'): And(UUID, Use(str)),
Optional('pid'): int,
Optional('pver'): int,
Optional('vstr'): And(bytes, Use(bytes.decode)),
Optional('pstr'): And(bytes, Use(bytes.decode)),
Optional('drive'): EepromDriveValue.encoder,
Optional('slew'): EepromSlewValue.encoder,
Optional('hysteresis'): EepromHysteresisValue.encoder,
Optional('back_power'): EepromBackPowerValue.encoder,
Optional('pins'): Or({}, {
int: {
Optional('function'): EepromPinFunctionValue.encoder,
Optional('pull'): EepromPinPullValue.encoder,
},
}),
})
json_decoder: ClassVar[JSONDecoder] = JSONDecoder()
json_encoder: ClassVar[JSONEncoder] = JSONEncoder()
def __init__(self, data=None, encoded=None, json=None, yaml=None):
self.data = data if data is not None else {}
if encoded is not None:
self.encoded = encoded
if json is not None:
self.json = json
if yaml is not None:
self.yaml = yaml
@property
def encoded(self):
"""Description encoded as serializable Python objects"""
return self.encoder.validate(self.data)
@encoded.setter
def encoded(self, value):
self.data = self.decoder.validate(value)
@property
def json(self):
"""Description encoded as JSON"""
return self.json_encoder.encode(self.encoded)
@json.setter
def json(self, value):
self.encoded = self.json_decoder.decode(value)
@property
def yaml(self):
"""Description encoded as YAML"""
return safe_dump(self.encoded, sort_keys=False)
@yaml.setter
def yaml(self, value):
self.encoded = safe_load(value)
def apply(self, eeprom):
"""Apply description to EEPROM"""
if 'uuid' in self.data:
eeprom.uuid = self.data['uuid']
if 'pid' in self.data:
eeprom.pid = self.data['pid']
if 'pver' in self.data:
eeprom.pver = self.data['pver']
if 'vstr' in self.data:
eeprom.vstr = self.data['vstr']
if 'pstr' in self.data:
eeprom.pstr = self.data['pstr']
if 'drive' in self.data:
eeprom.bank.drive = self.data['drive']
if 'slew' in self.data:
eeprom.bank.slew = self.data['slew']
if 'hysteresis' in self.data:
eeprom.bank.hysteresis = self.data['hysteresis']
if 'back_power' in self.data:
eeprom.power.back_power = self.data['back_power']
for i, pin in self.data.get('pins', {}).items():
eeprom.pins[i].used = True
if 'function' in pin:
eeprom.pins[i].function = pin['function']
if 'pull' in pin:
eeprom.pins[i].pull = pin['pull']
def describe(self, eeprom):
"""Describe EEPROM"""
self.data = {}
if eeprom.uuid.int:
self.data['uuid'] = eeprom.uuid
if eeprom.pid:
self.data['pid'] = eeprom.pid
if eeprom.pver:
self.data['pver'] = eeprom.pver
if eeprom.vstr:
self.data['vstr'] = eeprom.vstr
if eeprom.pstr:
self.data['pstr'] = eeprom.pstr
if eeprom.bank.drive:
self.data['drive'] = eeprom.bank.drive
if eeprom.bank.slew:
self.data['slew'] = eeprom.bank.slew
if eeprom.bank.hysteresis:
self.data['hysteresis'] = eeprom.bank.hysteresis
if eeprom.power.back_power:
self.data['back_power'] = eeprom.power.back_power
for i, pin in enumerate(eeprom.pins):
if pin.used:
self.data.setdefault('pins', {})
self.data['pins'][i] = {'function': pin.function}
if pin.pull:
self.data['pins'][i]['pull'] = pin.pull
@classmethod
def from_eeprom(cls, eeprom):
"""Construct description from EEPROM"""
self = cls()
self.describe(eeprom)
return self
def main(argv, session):
if six.PY2:
args = docopt(__doc__.encode('utf-8'), argv=argv)
else:
args = docopt(__doc__, argv=argv)
ERRORS = False
# Validate args.
s = Schema({
str:
Use(bool),
'<identifier>':
Or(
None,
And(str,
validate_ia_identifier,
error=
('<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, periods ".", '
'underscores "_", or dashes "-". However, <identifier> cannot begin '
'with periods, underscores, or dashes.'))),
'<file>':
And(
Use(lambda l: l
if not six.PY2 else convert_str_list_to_unicode(l)),
And(lambda f: all(os.path.exists(x) for x in f if x != '-'),
error='<file> should be a readable file or directory.'),
And(lambda f: False
if f == ['-'] and not args['--remote-name'] else True,
error=
'--remote-name must be provided when uploading from stdin.')),
'--remote-name':
Or(
None,
Use(lambda x: x.decode(sys.getfilesystemencoding())
if six.PY2 else x)),
'--spreadsheet':
Or(None,
os.path.isfile,
error='--spreadsheet should be a readable file.'),
'--file-metadata':
Or(None,
os.path.isfile,
error='--file-metadata should be a readable file.'),
'--metadata':
Or(None,
And(Use(get_args_dict), dict),
error='--metadata must be formatted as --metadata="key:value"'),
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--retries':
Use(lambda x: int(x[0]) if x else 0),
'--sleep':
Use(lambda l: int(l[0]), error='--sleep value must be an integer.'),
'--size-hint':
Or(Use(lambda l: str(l[0]) if l else None),
int,
None,
error='--size-hint value must be an integer.'),
'--status-check':
bool,
})
try:
args = s.validate(args)
except SchemaError as exc:
print('{0}\n{1}'.format(str(exc), printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Make sure the collection being uploaded to exists.
collection_id = args['--metadata'].get('collection')
if collection_id and not args['--no-collection-check'] and not args[
'--status-check']:
if isinstance(collection_id, list):
collection_id = collection_id[0]
collection = session.get_item(collection_id)
if not collection.exists:
print('You must upload to a collection that exists. '
'"{0}" does not exist.\n{1}'.format(
collection_id, printable_usage(__doc__)),
file=sys.stderr)
sys.exit(1)
# Status check.
if args['--status-check']:
if session.s3_is_overloaded():
print('warning: {0} is over limit, and not accepting requests. '
'Expect 503 SlowDown errors.'.format(args['<identifier>']),
file=sys.stderr)
sys.exit(1)
else:
print('success: {0} is accepting requests.'.format(
args['<identifier>']))
sys.exit()
elif args['<identifier>']:
item = session.get_item(args['<identifier>'])
# Upload keyword arguments.
if args['--size-hint']:
args['--header']['x-archive-size-hint'] = args['--size-hint']
# Upload with backups turned on by default.
if not args['--header'].get('x-archive-keep-old-version'):
args['--header']['x-archive-keep-old-version'] = '1'
queue_derive = True if args['--no-derive'] is False else False
verbose = True if args['--quiet'] is False else False
if args['--file-metadata']:
try:
args['<file>'] = json.load(open(args['--file-metadata']))
except json.decoder.JSONDecodeError:
args['<file>'] = list()
for line in open(args['--file-metadata']):
j = json.loads(line.strip())
args['<file>'].append(j)
upload_kwargs = dict(
metadata=args['--metadata'],
headers=args['--header'],
debug=args['--debug'],
queue_derive=queue_derive,
verbose=verbose,
verify=args['--verify'],
checksum=args['--checksum'],
retries=args['--retries'],
retries_sleep=args['--sleep'],
delete=args['--delete'],
)
# Upload files.
if not args['--spreadsheet']:
if args['-']:
local_file = TemporaryFile()
local_file.write(sys.stdin.read())
local_file.seek(0)
else:
local_file = args['<file>']
if isinstance(local_file,
(list, tuple, set)) and args['--remote-name']:
local_file = local_file[0]
if args['--remote-name']:
files = {args['--remote-name']: local_file}
else:
files = local_file
for _r in _upload_files(item, files, upload_kwargs):
if args['--debug']:
break
if (not _r.status_code) or (not _r.ok):
ERRORS = True
else:
if args['--open-after-upload']:
webbrowser.open_new_tab('{}//{}/details/{}'.format(
session.protocol, session.host, item.identifier))
# Bulk upload using spreadsheet.
else:
# Use the same session for each upload request.
with io.open(args['--spreadsheet'], 'rU', newline='',
encoding='utf-8') as csvfp:
spreadsheet = csv.DictReader(csvfp)
prev_identifier = None
for row in spreadsheet:
upload_kwargs_copy = deepcopy(upload_kwargs)
local_file = row['file']
identifier = row.get('item', row.get('identifier'))
if not identifier:
print('error: no identifier column on spreadsheet!')
sys.exit(1)
del row['file']
if 'identifier' in row:
del row['identifier']
elif 'item' in row:
del row['item']
if (not identifier) and (prev_identifier):
identifier = prev_identifier
item = session.get_item(identifier)
# TODO: Clean up how indexed metadata items are coerced
# into metadata.
md_args = [
'{0}:{1}'.format(k.lower(), v) for (k, v) in row.items()
if v
]
metadata = get_args_dict(md_args)
upload_kwargs_copy['metadata'].update(metadata)
r = _upload_files(item, local_file, upload_kwargs_copy,
prev_identifier, session)
for _r in r:
if args['--debug']:
break
if (not _r) or (not _r.ok):
ERRORS = True
else:
if args['--open-after-upload']:
webbrowser.open_new_tab('{}//{}/details/{}'.format(
session.protocol, session.host, identifier))
prev_identifier = identifier
if ERRORS:
sys.exit(1)
def main(argv, session: ArchiveSession) -> None:
args = docopt(__doc__, argv=argv)
# Validation error messages.
invalid_id_msg = (
'<identifier> should be between 3 and 80 characters in length, and '
'can only contain alphanumeric characters, underscores ( _ ), or '
'dashes ( - )')
# Validate args.
s = Schema({
str:
Use(bool),
'<file>':
list,
'--format':
list,
'--header':
Or(None,
And(Use(get_args_dict), dict),
error='--header must be formatted as --header="key:value"'),
'--glob':
list,
'delete':
bool,
'--retries':
Use(lambda i: int(i[0])),
'<identifier>':
str,
})
try:
args = s.validate(args)
except SchemaError as exc:
print(f'{exc}\n{printable_usage(__doc__)}', file=sys.stderr)
sys.exit(1)
verbose = True if not args['--quiet'] else False
item = session.get_item(args['<identifier>'])
if not item.exists:
print('{0}: skipping, item does\'t exist.', file=sys.stderr)
# Files that cannot be deleted via S3.
no_delete = ['_meta.xml', '_files.xml', '_meta.sqlite']
# Add keep-old-version by default.
if not args['--header'].get(
'x-archive-keep-old-version') and not args['--no-backup']:
args['--header']['x-archive-keep-old-version'] = '1'
if verbose:
print(f'Deleting files from {item.identifier}', file=sys.stderr)
if args['--all']:
files = list(item.get_files())
args['--cascade'] = True
elif args['--glob']:
files = item.get_files(glob_pattern=args['--glob'])
elif args['--format']:
files = item.get_files(formats=args['--format'])
else:
fnames = []
if args['<file>'] == ['-']:
fnames = [f.strip() for f in sys.stdin]
else:
fnames = [f.strip() for f in args['<file>']]
files = list(item.get_files(fnames))
if not files:
print(' warning: no files found, nothing deleted.', file=sys.stderr)
sys.exit(1)
errors = False
for f in files:
if not f:
if verbose:
print(f' error: "{f.name}" does not exist', file=sys.stderr)
errors = True
if any(f.name.endswith(s) for s in no_delete):
continue
if args['--dry-run']:
print(f' will delete: {item.identifier}/{f.name}', file=sys.stderr)
continue
try:
resp = f.delete(verbose=verbose,
cascade_delete=args['--cascade'],
headers=args['--header'],
retries=args['--retries'])
except requests.exceptions.RetryError as e:
print(f' error: max retries exceeded for {f.name}',
file=sys.stderr)
errors = True
continue
if resp.status_code != 204:
errors = True
msg = get_s3_xml_text(resp.content)
print(f' error: {msg} ({resp.status_code})', file=sys.stderr)
continue
if errors is True:
sys.exit(1)
def get_options(cls):
return {
"name":
str,
ConfigOption("description", default=None):
Or(str, None),
ConfigOption("logger_level", default=logger.TEST_INFO):
int,
ConfigOption("file_log_level", default=logger.DEBUG):
Or(int, None),
ConfigOption("runpath", default=default_runpath):
Or(None, str, lambda x: callable(x)),
ConfigOption("path_cleanup", default=True):
bool,
ConfigOption("all_tasks_local", default=False):
bool,
ConfigOption("shuffle", default=[]):
list, # list of string choices
ConfigOption("shuffle_seed",
default=float(random.randint(1, 9999))):
float,
ConfigOption("exporters", default=None):
Use(get_exporters),
ConfigOption("stdout_style", default=defaults.STDOUT_STYLE):
Style,
ConfigOption("report_dir", default=defaults.REPORT_DIR):
Or(str, None),
ConfigOption("xml_dir", default=None):
Or(str, None),
ConfigOption("pdf_path", default=None):
Or(str, None),
ConfigOption("json_path", default=None):
Or(str, None),
ConfigOption("http_url", default=None):
Or(str, None),
ConfigOption("pdf_style", default=defaults.PDF_STYLE):
Style,
ConfigOption("report_tags", default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption("report_tags_all", default=[]):
[Use(tagging.validate_tag_value)],
ConfigOption("merge_scheduled_parts", default=False):
bool,
ConfigOption("browse", default=False):
bool,
ConfigOption("ui_port", default=None):
Or(None, int),
ConfigOption(
"web_server_startup_timeout",
default=defaults.WEB_SERVER_TIMEOUT,
):
int,
ConfigOption("test_filter", default=filtering.Filter()):
filtering.BaseFilter,
ConfigOption("test_sorter", default=ordering.NoopSorter()):
ordering.BaseSorter,
# Test lister is None by default, otherwise Testplan would
# list tests, not run them
ConfigOption("test_lister", default=None):
Or(None, listing.BaseLister),
ConfigOption("verbose", default=False):
bool,
ConfigOption("debug", default=False):
bool,
ConfigOption("timeout", default=defaults.TESTPLAN_TIMEOUT):
Or(None, And(int, lambda t: t >= 0)),
ConfigOption("abort_wait_timeout", default=60):
int,
ConfigOption("interactive_handler", default=TestRunnerIHandler):
object,
ConfigOption("reset_report_uid", default=True):
bool,
ConfigOption("extra_deps", default=[]):
list,
}
from schema import Schema, And, Use, Optional, SchemaError
import os
from src.models import LiteModel
from .defaults import default_dir
model_schema = Schema(
And(str, os.path.exists, Use(LiteModel.load_from_filepath)))
def compare_shapes(s1, s2):
return list(s1) == list(s2)
def get_model_shape_schema(input_shape, output_shape):
input_schema = And(
lambda model: compare_shapes(model.input_shape, input_shape),
error=f"Got input shape {{0.input_shape}} expected {input_shape}")
output_schema = And(
lambda model: compare_shapes(model.output_shape, output_shape),
error=f"Got input shape {{0.output_shape}} expected {output_shape}")
return And(input_schema, output_schema)
models_schema = Schema({
"characters":
And(model_schema, get_model_shape_schema((1, 36, 36, 3), (1, 26))),
"bonuses":
And(model_schema, get_model_shape_schema((1, 15, 22, 3), (1, 5))),
"values":
And(model_schema, get_model_shape_schema((1, 17, 22, 3), (1, 22))),
)
_python_version = Schema(_pyversion, error="Invalid python version")
_requires_python = Schema(
lambda x: version.VersionPredicate("python (%s)" % x),
error="Invalid requires_python,"
)
_version_predicate = Schema(_pred_validate, error="Invalid version predicate")
_name = And(basestring, len, _no_slashes)
_version = And(basestring, len)
_action = Schema(And(basestring, len, Or(
"new release",
"remove",
"create",
"docupdate",
"update",
lambda x: bool(
x.split(" ", 1)[0] == "update" and
[y.strip() for y in x.split(" ", 1)[1].split(",") if y],
),
from .adapters import ADAPTERS
from ipaddress import ip_address, IPv4Address, IPv6Address
def is_ipv4(s):
return isinstance(ip_address(s), IPv4Address)
def is_ipv6(s):
return isinstance(ip_address(s), IPv6Address)
replace_schema = {
'from': str,
'to': str,
Optional('ttl'): And(int, lambda n: n >= 120),
}
config_schema = Schema({
'provider': lambda p: p in ADAPTERS.keys(),
'auth': {
Optional('cloudflare'): {
Optional('token'): str,
Optional('email'): str,
Optional('certtoken'): str,
Optional('debug'): bool,
},
},
'replace': {
Optional('a'): [{
**replace_schema, 'from': is_ipv4,
