class RouteInfo(object):
route_key = attr.attrib(default=None)
attributes_required = attr.attrib(default=False)
group_attributes_required = attr.attrib(default=False)
class StatusDetails(object):
known = attrib(default=None)
flaky = attrib(default=None)
message = attrib(default=None)
trace = attrib(default=None)
class Trainer:
"""The trainer object handles the actual training and testing of the model.
Args:
model: The PyTorch-Model
data_bundle: The training and test data as a DataBundle
optimizer_bundle: Contains the optimizer and the learning rate scheduler
run_id: A string identifying the specific run.
batch_size: The batch_size to train the model on.
epochs: The (total) number of epochs to train the model.
criterion: The optimization criterionn (loss), default is cross-entropy
metrics: A list of metric-object
device: The compute device or list of compute devices to place the model(s) on
logs_dir: The directory to store the results
conv_method: The strategy for handling convolutional layers for saturation computation
device_sat: The device to compute the saturation on. If None, the same device is used as for
the model.
delta: The delta threshold for computing saturation
data_parallel: Enable or Disable multi-GPU
downsampling: If None, downsampling is disabled, else the feature maps will be downsampled
to (downsampling x downsampling) resolution
"""
# private internal variables
_tracker: CheckLayerSat = attrib(init=False)
_save_path: str = attrib(init=False)
_initial_epoch: int = attrib(init=False)
_trained_epochs: int = attrib(init=False)
_experiment_done: bool = attrib(init=False)
# General Training setup
model: Module
data_bundle: DataBundle
optimizer_bundle: OptimizerSchedulerBundle
run_id: str
batch_size: int = 32
epochs: int = 30
criterion: nn.modules.loss._Loss = nn.modules.CrossEntropyLoss()
metrics: List[Metric] = attrib(factory=list)
# Technical Setup
device: str = 'cpu'
logs_dir: str = './logs'
# delve setup
conv_method = 'channelwise'
device_sat: Optional[str] = None
delta: float = 0.99
data_parallel: bool = False
downsampling: Optional[int] = None
def _initialize_tracker(self):
writer = CSVandPlottingWriter(self._save_path.replace('.csv', ''),
primary_metric='test_accuracy')
self._tracker = CheckLayerSat(
self._save_path.replace('.csv', ''), [writer],
self.model,
ignore_layer_names='convolution',
stats=['lsat', 'idim'],
sat_threshold=self.delta,
verbose=False,
conv_method=self.conv_method,
log_interval=1,
device=self.device_sat,
reset_covariance=True,
max_samples=None,
initial_epoch=self._initial_epoch,
interpolation_strategy='nearest'
if self.downsampling is not None else None,
interpolation_downsampling=self.downsampling)
def _initialize_saving_structure(self):
save_dir: str = build_saving_structure(
logs_dir=self.logs_dir,
model_name=self.model.name,
dataset_name=self.data_bundle.dataset_name,
output_resolution=self.data_bundle.output_resolution,
run_id=self.run_id)
self._save_path = os.path.join(
save_dir,
f"{self.model.name}-{self.data_bundle.dataset_name}-r{self.data_bundle.output_resolution}-bs{self.batch_size}-e{self.epochs}.csv"
)
def _load_model(self):
self.model.load_state_dict(
torch.load(self._save_path.replace('.csv',
'.pt'))['model_state_dict'])
if self.data_parallel:
# TODO: make this work with DistributedDataParallel
self.model = nn.DataParallel(self.model)
# from torch.nn.parallel import DistributedDataParallel
self.model = self.model.to(self.device)
def _load_optimizer_and_scheduler(self):
self.optimizer_bundle.optimizer.load_state_dict(
torch.load(self._save_path.replace('.csv', '.pt'))['optimizer'])
if self.optimizer_bundle.scheduler is not None:
self.optimizer_bundle.scheduler.load_state_dict(
torch.load(self._save_path.replace('.csv',
'.pt'))['scheduler'])
def _load_initial_and_trained_epoch(self):
self._trained_epochs = torch.load(
self._save_path.replace('.csv', '.pt'))['epoch']
self._initial_epoch = self._trained_epochs + 1
def _check_training_done(self):
if self._initial_epoch >= self.epochs:
self._experiment_done = True
print(
f'Experiment Logs for the exact same experiment with identical run_id was detected, '
f'training will be skipped, consider using another run_id')
def _checkpointing(self):
self._initial_epoch = 0
self._trained_epochs = 0
self._experiment_done = False
self.model = self.model.to(self.device)
if os.path.exists(self._save_path):
self._load_initial_and_trained_epoch()
self._check_training_done()
self._load_model()
self._load_optimizer_and_scheduler()
print('Resuming existing run, starting at epoch',
self._initial_epoch + 1, 'from',
self._save_path.replace('.csv', '.pt'))
def _enable_benchmark_mode_if_cuda(self):
if "cuda" in self.device:
from torch.backends import cudnn
cudnn.benchmark = True
def __attrs_post_init__(self):
self.device_sat = self.device if self.device_sat is None else self.device_sat
self._enable_benchmark_mode_if_cuda()
self._initialize_saving_structure()
self._checkpointing()
self._initialize_tracker()
def _reset_metrics(self):
for metric in self.metrics:
metric.reset()
def _eval_metrics(self, y_true: torch.Tensor, y_pred: torch.Tensor):
for metric in self.metrics:
metric.update(y_true, y_pred)
def _print_status(self, batch: int, old_time: int, dataset: DataLoader):
metrics = [
f"{metric.name}: {round(metric.value, 3)}"
for metric in self.metrics
]
print(batch, 'of', len(dataset), 'processing time',
round(time() - old_time, 3), *metrics)
def _print_epoch_status(self, epoch: int, old_time: int,
metric_dict: Dict[str, float]):
metrics = [f"{k}: {round(v, 3)}" for (k, v) in metric_dict.items()]
print(epoch + 1, 'of', self.epochs, 'processing time',
round(time() - old_time, 3), *metrics)
def _track_results(self, prefix: str, metric_name: str,
metric_value: float) -> Tuple[str, float]:
self._tracker.add_scalar(f"{prefix}_{metric_name}", metric_value)
return f"{prefix}_{metric_name}", metric_value
def _track_metrics(self, prefix: str, loss: float,
total: int) -> Dict[str, float]:
result: Dict[str, float] = dict()
for metric in self.metrics:
name, val = self._track_results(prefix, metric.name, metric.value)
result[name] = val
name, val = self._track_results(prefix, "loss", loss / total)
result[name] = val
return result
def _save_checkpoint(self, train_metric: Dict[str, float],
test_metric: Dict[str, float], epoch: int):
state_dict = {
'model_state_dict':
self.model.state_dict(),
'optimizer':
self.optimizer_bundle.optimizer.state_dict(),
'scheduler':
None if self.optimizer_bundle.scheduler is None else
self.optimizer_bundle.scheduler.state_dict(),
'epoch':
epoch
}
state_dict.update(train_metric)
state_dict.update(test_metric)
torch.save(state_dict, self._save_path.replace('.csv', '.pt'))
def train(self):
"""Train the model.
The model is trained for a total number of epochs given the number of epochs provided in the constructor.
This includes epochs this model was trained previously.
Returns:
The path to the saturation ans metric logs.
"""
if self._experiment_done:
return
old_time = time()
for epoch in range(self._initial_epoch, self.epochs):
print('Start training epoch', epoch + 1)
train_metric = self.train_epoch()
test_metric = self.test()
train_metric.update(test_metric)
self._print_epoch_status(epoch=epoch,
old_time=old_time,
metric_dict=train_metric)
old_time = time()
if self.optimizer_bundle.scheduler is not None:
self.optimizer_bundle.scheduler.step()
self._tracker.add_saturations()
self._save_checkpoint(train_metric=train_metric,
test_metric=test_metric,
epoch=epoch)
self._tracker.close()
return self._save_path + '.csv'
def train_epoch(self) -> Dict[str, float]:
"""Train a single epoch.
Returns:
A dictionary containing all metrics computed incrementally during training.
"""
self.model.train()
self._reset_metrics()
running_loss = 0
total = 0
old_time = time()
for batch, data in enumerate(self.data_bundle.train_dataset):
if batch % 10 == 0 and batch != 0:
self._print_status(batch, old_time,
self.data_bundle.train_dataset)
old_time = time()
inputs, labels = data
inputs, labels = inputs.to(self.device), labels.to(self.device)
self.optimizer_bundle.optimizer.zero_grad()
outputs = self.model(inputs)
_, predicted = torch.max(outputs.data, 1)
self._eval_metrics(labels, outputs)
loss = self.criterion(outputs, labels)
loss.backward()
self.optimizer_bundle.optimizer.step()
running_loss += loss.item()
total += self.batch_size
return self._track_metrics('training', running_loss, total)
def test(self):
"""Evaluate the model on the test set.
Returns:
The metric computed on the test set.
"""
self._reset_metrics()
self.model.eval()
total = 0
test_loss = 0
with torch.no_grad():
old_time = time()
for batch, data in enumerate(self.data_bundle.test_dataset):
inputs, labels = data
inputs, labels = inputs.to(self.device), labels.to(self.device)
outputs = self.model(inputs)
loss = self.criterion(outputs, labels)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
test_loss += loss.item()
self._eval_metrics(labels, outputs)
if batch % 10 == 0 or batch == (
len(self.data_bundle.test_dataset) - 1):
self._print_status(batch, old_time,
self.data_bundle.test_dataset)
old_time = time()
test_metrics = self._track_metrics('test', test_loss, total)
return test_metrics
class TestStepResult(ExecutableItem):
id = attrib(default=None)
class Label(object):
name = attrib(default=None)
value = attrib(default=None)
class GoogleTrackBase(Track):
"""Adds an id argument."""
id = attrib(default=Factory(lambda: None))
class OfflineRenderDriver(object):
"""Obtain and store ancillary rendering parameters, and use them to perform file-to-file rendering."""
target_layout = attrib()
speakers_file = attrib()
output_gain_db = attrib()
fail_on_overload = attrib()
enable_block_duration_fix = attrib()
config = attrib(default=Factory(dict))
programme_id = attrib(default=None)
complementary_object_ids = attrib(default=Factory(list))
conversion_mode = attrib(default=None)
blocksize = 8192
@classmethod
def add_args(cls, parser):
"""Add arguments to an ArgumentParser that can be used by from_args."""
formats_string = ", ".join(bs2051.layout_names)
parser.add_argument(
"-s",
"--system",
required=True,
metavar="target_system",
help="Target output system, accoring to ITU-R BS.2051. "
"Available systems are: {}".format(formats_string))
parser.add_argument("-l",
"--layout",
type=argparse.FileType("r"),
metavar="layout_file",
help="Layout config file")
parser.add_argument("--output-gain-db",
type=float,
metavar="gain_db",
default=0,
help="output gain in dB (default: 0)")
parser.add_argument(
"--fail-on-overload",
"-c",
action="store_true",
help="fail if an overload condition is detected in the output")
parser.add_argument(
"--enable-block-duration-fix",
action="store_true",
help="automatically try to fix faulty block format durations")
parser.add_argument("--programme",
metavar="id",
help="select an audioProgramme to render by ID")
parser.add_argument(
"--comp-object",
metavar="id",
action="append",
default=[],
help="select an audioObject by ID from a complementary group")
parser.add_argument(
'--apply-conversion',
choices=("to_cartesian", "to_polar"),
help=
'Apply conversion to Objects audioBlockFormats before rendering')
@classmethod
def from_args(cls, args):
return cls(
target_layout=args.system,
speakers_file=args.layout,
output_gain_db=args.output_gain_db,
fail_on_overload=args.fail_on_overload,
enable_block_duration_fix=args.enable_block_duration_fix,
programme_id=args.programme,
complementary_object_ids=args.comp_object,
conversion_mode=args.apply_conversion,
)
def load_output_layout(self):
"""Load the specified layout.
Returns:
layout (Layout): loudspeaker layout
upmix (sparse array or None): optional matrix to apply after rendering
n_channels (int): number of channels required in output file
"""
spkr_layout = bs2051.get_layout(self.target_layout)
if self.speakers_file is not None:
real_layout = layout.load_real_layout(self.speakers_file)
spkr_layout, upmix = spkr_layout.with_real_layout(real_layout)
spkr_layout.check_positions()
spkr_layout.check_upmix_matrix(upmix)
upmix = scipy.sparse.csc_matrix(upmix.T)
n_channels = upmix.shape[1]
else:
upmix = None
n_channels = len(spkr_layout.channels)
return spkr_layout, upmix, n_channels
@property
def output_gain_linear(self):
return 10.0**(self.output_gain_db / 20.0)
@classmethod
def lookup_adm_element(cls, adm, element_id, element_type,
element_type_name):
"""Lookup an element in adm by type and ID, with nice error messages."""
if element_id is None:
return None
try:
element = adm[element_id]
except KeyError:
raise KeyError(
"could not find {element_type_name} with ID {element_id}".
format(
element_type_name=element_type_name,
element_id=element_id,
))
if not isinstance(element, element_type):
raise ValueError(
"{element_id} is not an {element_type_name}".format(
element_type_name=element_type_name,
element_id=element_id,
))
return element
def get_audio_programme(self, adm):
return self.lookup_adm_element(adm, self.programme_id, AudioProgramme,
"audioProgramme")
def get_complementary_objects(self, adm):
return [
self.lookup_adm_element(adm, obj_id, AudioObject, "audioObject")
for obj_id in self.complementary_object_ids
]
def apply_conversion(self, selected_items):
if self.conversion_mode is None:
return selected_items
elif self.conversion_mode == "to_cartesian":
return convert_objects_to_cartesian(selected_items)
elif self.conversion_mode == "to_polar":
return convert_objects_to_polar(selected_items)
else:
assert False
def get_rendering_items(self, adm):
"""Get rendering items from the input file adm.
Parameters:
adm (ADM): ADM to get the RenderingItems from
Returns:
list of RenderingItem: selected rendering items
"""
audio_programme = self.get_audio_programme(adm)
comp_objects = self.get_complementary_objects(adm)
selected_items = select_rendering_items(
adm,
audio_programme=audio_programme,
selected_complementary_objects=comp_objects)
selected_items = preprocess_rendering_items(selected_items)
selected_items = self.apply_conversion(selected_items)
return selected_items
def render_input_file(self, infile, spkr_layout, upmix=None):
"""Get sample blocks of the input file after rendering.
Parameters:
infile (Bw64AdmReader): file to read from
spkr_layout (Layout): layout to render to
upmix (sparse array or None): optional upmix to apply
Yields:
2D sample blocks
"""
renderer = Renderer(spkr_layout, **self.config)
renderer.set_rendering_items(self.get_rendering_items(infile.adm))
for input_samples in chain(infile.iter_sample_blocks(self.blocksize),
[None]):
if input_samples is None:
output_samples = renderer.get_tail(infile.sampleRate,
infile.channels)
else:
output_samples = renderer.render(infile.sampleRate,
input_samples)
output_samples *= self.output_gain_linear
if upmix is not None:
output_samples *= upmix
yield output_samples
def run(self, input_file, output_file):
"""Render input_file to output_file."""
spkr_layout, upmix, n_channels = self.load_output_layout()
output_monitor = PeakMonitor(n_channels)
with openBw64Adm(input_file) as infile:
infile.adm.validate()
timing_fixes.check_blockFormat_timings(
infile.adm, fix=self.enable_block_duration_fix)
formatInfo = FormatInfoChunk(formatTag=1,
channelCount=n_channels,
sampleRate=infile.sampleRate,
bitsPerSample=infile.bitdepth)
with openBw64(output_file, "w", formatInfo=formatInfo) as outfile:
for output_block in self.render_input_file(
infile, spkr_layout, upmix):
output_monitor.process(output_block)
outfile.write(output_block)
output_monitor.warn_overloaded()
if self.fail_on_overload and output_monitor.has_overloaded():
raise Exception("error: output overloaded")
class AggregationAuthorizationProperties(ResourceProperties):
AuthorizedAccountId = attrib(default=None)
AuthorizedAwsRegion = attrib(default=None)
class ConfigurationAggregatorProperties(ResourceProperties):
AccountAggregationSources = attrib(default=None)
ConfigurationAggregatorName = attrib(default=None)
OrganizationAggregationSource = attrib(default=None)
class ConfigurationRecorderProperties(ResourceProperties):
Name = attrib(default=None)
RecordingGroup = attrib(default=None)
RoleARN = attrib(default=None)
class DeliveryChannelProperties(ResourceProperties):
ConfigSnapshotDeliveryProperties = attrib(default=None)
Name = attrib(default=None)
S3BucketName = attrib(default=None)
S3KeyPrefix = attrib(default=None)
SnsTopicARN = attrib(default=None)
class PackagesNotFoundError(CondaException):
"""
Conda 4.5 exception - this reports all missing packages.
"""
packages = attrib(default=())
class UnknownCondaError(CondaException):
data = attrib(default=Factory(dict))
class CondaException(Exception, NonStrictAttrs):
command = attrib()
message = attrib(default=None)
class Monster(Combatant):
cr = attrib(default=0)
xp = attrib(default=0)
size = attrib(default="medium")
mtype = attrib(default="humanoid")
alignment = attrib(default="unaligned")
str = attrib(default=10)
dex = attrib(default=10)
con = attrib(default=10)
int = attrib(default=10)
wis = attrib(default=10)
cha = attrib(default=10)
def __getattr__(self, attr_name):
if attr_name[3:] == '_mod' and \
attr_name[:3] in ('str', 'dex', 'con', 'int', 'wis', 'cha'):
return self.ability_modifier(getattr(self, attr_name[:3]))
elif attr_name == 'initiative_mod':
return self.ability_modifier(getattr(self, 'dex'))
else:
raise AttributeError(
f"'Monster' object has no attribute '{attr_name}'")
def ability_modifier(self, stat):
return floor((stat - 10) / 2)
armor = attrib(default="")
speed = attrib(default=30)
skills = attrib(default=attr_factory(dict))
resist = attrib(default=attr_factory(list))
immune = attrib(default=attr_factory(list))
vulnerable = attrib(default=attr_factory(list))
languages = attrib(default=attr_factory(list))
features = attrib(default=attr_factory(dict))
actions = attrib(default=attr_factory(dict))
legendary_actions = attrib(default=attr_factory(dict))
reactions = attrib(default=attr_factory(dict))
notes = attrib(default="")
origin = attrib(default="origin unknown")
class PathRequirement(Requirement):
_name: str = attrib()
_path: str = attrib()
_extras: Set[str] = attrib()
_environment_markers: Optional[EnvironmentMarker] = attrib()
_logger: Logger = attrib()
def name(self) -> str:
return canonicalize_name(self._name)
def extras(self) -> Set[str]:
return self._extras
def logger(self) -> Logger:
return self._logger
def add(self, other: Requirement,
target_platform: TargetPlatform) -> Requirement:
if not self.applies_to_target(target_platform):
return other
elif not other.applies_to_target(target_platform):
return self
elif self.name() != other.name():
raise IncompatibleRequirements(
"Cannot add requirements with different names `{name1}` and `{name2}`"
.format(name1=self.name(), name2=other.name()))
else:
if isinstance(other, VersionRequirement):
return self
elif isinstance(other, UrlRequirement):
raise IncompatibleRequirements(
"Cannot combine requirements with path `{path} and url `{url}`"
.format(path=self.path, url=other.url))
elif isinstance(other, PathRequirement):
if self.path != other.path:
raise IncompatibleRequirements(
"Cannot combine requirements with different paths `{path1}` and `{path2}`"
.format(path1=self.path, path2=other.path))
else:
return self
else:
raise IncompatibleRequirements(
"Did not recognize requirement type of {}".format(other))
def source(self) -> PathSource:
return PathSource(path=self._path)
def environment_markers(self) -> Optional[EnvironmentMarker]:
return self._environment_markers
def to_line(self) -> str:
extras = "[" + ",".join(self.extras()) + "]" if self.extras() else ""
return "file://{path}#egg={name}{extras}".format(path=self._path,
extras=extras,
name=self.name())
def path(self) -> str:
return self._path
def change_path(self, mapping: Callable[[str], str]) -> "PathRequirement":
return evolve(self, path=mapping(self._path))
class ObjectRecognizerAsTemplateLearner(TemplateLearner):
_object_recognizer: ObjectRecognizer = attrib(
validator=instance_of(ObjectRecognizer))
_language_mode: LanguageMode = attrib(validator=instance_of(LanguageMode),
kw_only=True)
_concepts_to_templates: ImmutableSetMultiDict[Concept,
SurfaceTemplate] = attrib(
init=False)
def learn_from(
self,
language_perception_semantic_alignment:
LanguagePerceptionSemanticAlignment,
offset: int = 0,
) -> None:
# The object recognizer doesn't learn anything.
# It just recognizes predefined object patterns.
pass
@staticmethod
def _enrich_post_process(
perception_graph_after_matching: PerceptionGraph,
immutable_new_nodes: AbstractSet[SemanticNode],
) -> Tuple[PerceptionGraph, AbstractSet[SemanticNode]]:
new_nodes = []
perception_graph_after_processing = perception_graph_after_matching
for candiate_object_graph in extract_candidate_objects(
perception_graph_after_matching,
sort_by_increasing_size=False):
fake_pattern_graph = PerceptionGraphPattern.from_graph(
candiate_object_graph)
fake_object_semantic_node = ObjectSemanticNode(
concept=FunctionalObjectConcept("unknown_object"))
perception_graph_after_processing = replace_match_with_object_graph_node(
matched_object_node=fake_object_semantic_node,
current_perception=perception_graph_after_processing,
pattern_match=PerceptionGraphPatternMatch(
matched_pattern=fake_pattern_graph.
perception_graph_pattern,
graph_matched_against=perception_graph_after_processing,
matched_sub_graph=candiate_object_graph,
pattern_node_to_matched_graph_node=fake_pattern_graph.
perception_graph_node_to_pattern_node,
),
).perception_graph_after_replacement
new_nodes.append(fake_object_semantic_node)
return (
perception_graph_after_processing,
immutableset(chain(immutable_new_nodes, new_nodes)),
)
def enrich_during_learning(
self, language_perception_semantic_alignment:
LanguagePerceptionSemanticAlignment
) -> LanguagePerceptionSemanticAlignment:
return self._object_recognizer.match_objects_with_language(
language_perception_semantic_alignment,
post_process=self._enrich_post_process)
def enrich_during_description(
self, perception_semantic_alignment: PerceptionSemanticAlignment
) -> PerceptionSemanticAlignment:
(new_perception_semantic_alignment,
_) = self._object_recognizer.match_objects(
perception_semantic_alignment,
post_process=self._enrich_post_process)
return new_perception_semantic_alignment
def templates_for_concept(
self, concept: Concept) -> ImmutableSet[SurfaceTemplate]:
if self._language_mode == LanguageMode.ENGLISH:
return self._concepts_to_templates[concept]
elif self._language_mode == LanguageMode.CHINESE:
if concept.debug_string == "you":
return immutableset([
SurfaceTemplate.for_object_name(
"ni3", language_mode=self._language_mode)
])
if concept.debug_string == "me":
return immutableset([
SurfaceTemplate.for_object_name(
"wo3", language_mode=self._language_mode)
])
mappings = (
GAILA_PHASE_1_CHINESE_LEXICON._ontology_node_to_word # pylint:disable=protected-access
)
for k, v in mappings.items():
if k.handle == concept.debug_string:
return immutableset([
SurfaceTemplate.for_object_name(
v.base_form, language_mode=self._language_mode)
])
# FunctionalObjectConcepts mean we have recognized an object but don't have
# Knowledge of what the lexicalization is. So we just return an empty set
if isinstance(concept, FunctionalObjectConcept):
return immutableset()
raise RuntimeError(f"Invalid concept {concept}")
def log_hypotheses(self, log_output_path: Path) -> None:
for concept, hypothesis in self.concepts_to_patterns().items():
hypothesis.render_to_file(
graph_name="perception",
output_file=Path(log_output_path /
f"{str(type(self))}-{concept.debug_string}"),
)
def concepts_to_patterns(self) -> Dict[Concept, PerceptionGraphPattern]:
return {
k: v
for k, v in
self._object_recognizer._concepts_to_static_patterns.items() # pylint:disable=protected-access
}
@_concepts_to_templates.default
def _init_concepts_to_templates(
self) -> ImmutableSetMultiDict[Concept, SurfaceTemplate]:
# Ground is added explicitly to this list because the code
# Which matches the ground matches by recognition and not shape
# See: `ObjectRecognizer.match_objects`
return immutablesetmultidict((
concept,
SurfaceTemplate.for_object_name(name,
language_mode=self._language_mode),
) for (concept, name) in (
list(self._object_recognizer._concepts_to_names.items() # pylint:disable=protected-access
) + [(GROUND_OBJECT_CONCEPT, "ground")]))
class VersionRequirement(Requirement):
_name: str = attrib()
_versions: List[Tuple[str, str]] = attrib()
_extras: Set[str] = attrib()
_environment_markers: Optional[EnvironmentMarker] = attrib()
_logger: Logger = attrib()
def name(self) -> str:
return canonicalize_name(self._name)
def extras(self) -> Set[str]:
return self._extras
def logger(self) -> Logger:
return self._logger
def add(self, other: Requirement,
target_platform: TargetPlatform) -> Requirement:
if not self.applies_to_target(target_platform):
return other
elif not other.applies_to_target(target_platform):
return self
elif self.name() != other.name():
raise IncompatibleRequirements(
"Cannot add requirments with different names `{name1}` and `{name2}`"
.format(name1=self.name(), name2=other.name()))
else:
if isinstance(other, PathRequirement):
return other
elif isinstance(other, UrlRequirement):
return other
elif isinstance(other, VersionRequirement):
return VersionRequirement(
name=self.name(),
extras=self._extras.union(other._extras),
versions=self.version() + other.version(),
environment_markers=None,
logger=self.logger(),
)
else:
raise IncompatibleRequirements(
"Did not recognize requirement type of {}".format(other))
def source(self) -> None:
return None
def environment_markers(self) -> Optional[EnvironmentMarker]:
return self._environment_markers
def version(self) -> List[Tuple[str, str]]:
return self._versions
def to_line(self) -> str:
version = ", ".join([
"{operator} {specifier}".format(operator=operator,
specifier=specifier)
for operator, specifier in self._versions
])
extras = ("[{extras}]".format(
extras=",".join(self.extras())) if self.extras() else "")
return "{name}{extras} {version}".format(name=self._name,
version=version,
extras=extras)
class CH4(CH5):
_name = attrib(init=False, default='Chain Heal (Rank 4)')
_base = attrib(init=False, default=(605, 692))
class UrlRequirement(Requirement):
_name: str = attrib()
_url: str = attrib()
_extras: Set[str] = attrib()
_environment_markers: Optional[EnvironmentMarker] = attrib()
_logger: Logger = attrib()
def name(self) -> str:
return canonicalize_name(self._name)
def extras(self) -> Set[str]:
return self._extras
def logger(self) -> Logger:
return self._logger
def add(self, other: Requirement,
target_platform: TargetPlatform) -> Requirement:
if not self.applies_to_target(target_platform):
return other
elif not other.applies_to_target(target_platform):
return self
elif self.name() != other.name():
raise IncompatibleRequirements(
"Cannot add requirments with different names `{name1}` and `{name2}`"
.format(name1=self.name(), name2=other.name()))
else:
if isinstance(other, VersionRequirement):
return self
elif isinstance(other, PathRequirement):
raise IncompatibleRequirements(
"Cannot combine requirements with with url `{url}` and path `{path}`"
.format(url=self.url, path=other.path))
elif isinstance(other, UrlRequirement):
if self.url != other.url:
raise IncompatibleRequirements(
"Cannot combine requirements with different urls `{url1}` and `{url2}`"
.format(url1=self.url, url2=other.url))
else:
return self
else:
raise IncompatibleRequirements(
"Did not recognize requirement type of {}".format(other))
def source(self) -> PackageSource:
if self._url.startswith("git+"):
return self._handle_git_source(self._url[4:])
elif self._url.startswith("git://"):
return self._handle_git_source(self._url)
elif self._url.startswith("hg+"):
return self._handle_hg_source(self._url[3:])
elif self.url_scheme() == "file":
return PathSource(path=self.url_path())
else:
return UrlSource(url=self._url, logger=self._logger)
def environment_markers(self) -> Optional[EnvironmentMarker]:
return self._environment_markers
def _handle_hg_source(self, url: str) -> HgSource:
try:
url, rev = url.split("@")
except ValueError:
return HgSource(url=url, logger=self._logger)
else:
return HgSource(url=url, revision=rev, logger=self._logger)
def _handle_git_source(self, url: str) -> GitSource:
try:
url, rev = url.split("@")
except ValueError:
return GitSource(url=url, logger=self._logger)
else:
return GitSource(url=url, logger=self._logger, revision=rev)
def to_line(self) -> str:
extras = "[" + ",".join(self.extras()) + "]" if self.extras() else ""
return "{url}#egg={name}{extras}".format(url=self._url,
name=self.name(),
extras=extras)
def url(self) -> str:
return self._url
def url_scheme(self) -> str:
url = urlparse(self.url())
return url.scheme
def url_path(self) -> str:
url = urlparse(self.url())
return url.path
class ExecutableItem(object):
name = attrib(default=None)
status = attrib(default=None)
statusDetails = attrib(default=None)
stage = attrib(default=None)
description = attrib(default=None)
descriptionHtml = attrib(default=None)
steps = attrib(default=Factory(list))
attachments = attrib(default=Factory(list))
parameters = attrib(default=Factory(list))
start = attrib(default=None)
stop = attrib(default=None)
class DataDictionary:
"""The parent class of a container that provides information on a tabular data structure."""
__columns__ = attrib(init=False, repr=False)
@property
def columns(self):
"""Show column names."""
return self.__columns__
def list_columns(self):
"""List all columns."""
return [getattr(self, col) for col in self.__columns__]
# def __repr__(self):
# name = self.__class__.__qualname__
# cols = "\n\t".join([repr(getattr(self, col)) for col in self.__columns__])
# return f"{name}[DataDictionary]\n\t{cols}\n"
def _cols_valid(self, dataframe: pd.DataFrame):
"""Test match of columns in datadictionary vs dataframe."""
results, msgs = [], []
col_dict = {col.name: col for col in self.list_columns()}
cols = list(dict.fromkeys(list(col_dict) + list(dataframe.columns)))
for col in cols:
dd_col = col_dict.get(col, None)
df_col = dataframe.get(col, None)
if dd_col is None:
results.append(False)
msg = f"The column [{col}] is missing from the DataDictionary but is in the dataframe."
msgs.append(msg)
if df_col is None:
results.append(False)
msg = f"The column [{col}] is missing from the dataframe but is in the DataDictionary."
msgs.append(msg)
if dd_col is not None and df_col is not None:
results.append(True)
return results, msgs
def _types_valid(self, dataframe: pd.DataFrame):
"""Test column types in dataframe."""
results, msgs = [], []
for dd_col in self.list_columns():
df_col = dataframe.get(dd_col.name, None)
if df_col is not None:
test_eq = dd_col.dtype.value == df_col.dtype.name
if test_eq is False:
msg = f"The column [{dd_col.name}] in the DataDictionary has type [{dd_col.dtype.value}] "
msg += f"which is different from type in the dataframe passed [{df_col.dtype.name}]."
msgs.append(msg)
results.append(test_eq)
return results, msgs
def _validators_valid(self, dataframe: pd.DataFrame):
"""Test validators for columns in dataframe."""
results, msgs = [], []
for dd_col in self.list_columns():
df_col = dataframe.get(dd_col.name, None)
if df_col is not None and len(dd_col.validator) > 0:
for validator in dd_col.validator:
try:
validator(inst=self, attr=dd_col, value=df_col)
results.append(True)
except:
results.append(False)
msgs.append(
f"The column [{dd_col.name}] failed {str(validator)[1:-1]}."
)
return results, msgs
def validate(
self, dataframe: pd.DataFrame, types: bool = True, validators: bool = True
):
"""Validate passed dataframe types and/or validators against DataDictionary.
:param pd.DataFrame dataframe: The dataframe to validate.
:param bool types: Test DataDictionary types against dataframe. Default is True.
:param bool validators: Test DataDictionary validators against dataframe. Default is True.
:return: True if test pass.
:rtype: bool
:raises DataDictionaryValidateError: If any of the following rules are broken -
- All columns present in the data dictionary are in the dataframe and vice versa.
- If types is True, test the data types of the dataframe match the data dictionary.
- If validators is True, test that any attached validator pass for a column.
"""
rets, msgs = self._cols_valid(dataframe)
def _update_ret_msg(fn, dataframe):
ret, msg = fn(dataframe)
rets.extend(ret)
msgs.extend(msg)
if types is True:
_update_ret_msg(self._types_valid, dataframe)
if validators is True:
_update_ret_msg(self._validators_valid, dataframe)
if not all(rets):
msg = "\n".join(msgs)
raise DataDictionaryValidateError("\n" + msg)
return True
def def_parameter(
self,
column: str,
add_dtype_validator: bool = False,
dtype_mapping: Mapping = PANDAS_DTYPE_CONVERSION,
):
"""Define a parameter for a model from a data dictioanry column.
:param str column: The column name.
:param bool add_type_validator: If True, an instance_of validator is added to the parameter.
:param Mapping dtype_mapping: A conversion mapping of PandasDtypes to different types.
"""
col = getattr(self, column, None)
if col is None:
msg = f"The column [{column}] does not belong to the data dictionary."
raise AttributeError(msg)
dtype = dtype_mapping.get(col.dtype)
validator = col.validator.copy()
if add_dtype_validator is True:
validator.append(instance_of(dtype))
return def_parameter(
dtype=dtype, description=col.description, validator=validator,
)
def def_sensitivity(self, column):
raise NotImplementedError("This feature is not implemented yet.")
def def_meta(self, column):
raise NotImplementedError("This feature is not implemented yet.")
def def_column(self, column):
"""Define a column for another data dictionary using an existing data dictioanry column.
:param str column: The column name.
"""
col = getattr(self, column, None)
if col is None:
msg = f"The column [{column}] does not belong to the data dictionary."
raise AttributeError(msg)
return col
class Parameter(object):
name = attrib(default=None)
value = attrib(default=None)
class Encounter:
name = attrib(default="")
location = attrib(default="")
notes = attrib(default="")
groups = attrib(default=[])
class Link(object):
type = attrib(default=None)
url = attrib(default=None)
name = attrib(default=None)
class Combatant:
name = attrib(default="")
race = attrib(default="")
ac = attrib(default=0)
senses = attrib(default=attr_factory(dict))
conditions = attrib(default=attr_factory(dict))
_max_hp = attrib(default=10)
_cur_hp = attrib(default=10)
@property
def max_hp(self):
return self._max_hp
@max_hp.setter
def max_hp(self, value):
try:
self._max_hp = int(value)
except ValueError:
# we have an expression for max hp, so roll it
self._max_hp = dice.roll_dice_expr(value)
# setting max_hp for the first time? we should set cur_hp too
if self.cur_hp is None:
self.cur_hp = self._max_hp
@property
def cur_hp(self):
return self._cur_hp
@cur_hp.setter
def cur_hp(self, value):
if value > self.max_hp:
value = self.max_hp
if value < 0:
if abs(value) >= self.max_hp:
self.conditions = {'dead': inf}
value = 0
self._cur_hp = value
def set_condition(self, condition, duration=inf):
self.conditions[condition] = duration
def unset_condition(self, condition):
try:
self.conditions.pop(condition)
except KeyError:
# We can probably safely ignore failures here,
# since it shouldn't be the end of the world
# to remove a condition that isn't in effect.
pass
def decrement_condition_durations(self):
conditions_removed = []
for condition in list(self.conditions):
self.conditions[condition] -= 1
if self.conditions[condition] == 0:
self.conditions.pop(condition)
conditions_removed.append(condition)
return conditions_removed
class Attachment(object):
name = attrib(default=None)
source = attrib(default=None)
type = attrib(default=None)
class Character(Combatant):
cclass = attrib(default="Fighter")
level = attrib(default=1)
initiative_mod = attrib(default=0)
class SubspaceRulesClassifier(BaseEstimator):
rules: List[Rule] = attrib()
max_depth: int = attrib()
random_state: int = attrib(default=42)
train_default_on_whole_set: bool = attrib(default=True)
depth_strategy: DepthStrategy = attrib(
default=DepthStrategy.SUBTRACT_FROM_RULES)
_clf_by_rule: Dict[Rule, any] = attrib(init=False, factory=dict)
_default_clf: any = attrib(init=False)
def fit(self, x, y):
x, y = check_X_y(x, y)
already_covered_indicies = set()
for rule in self.rules:
if self.depth_strategy == DepthStrategy.SUBTRACT_FROM_RULES:
depth = self.max_depth - len(rule.statements)
else:
depth = self.max_depth
matching_sample_indicies = pipe(
x,
map(to_instance),
enumerate,
filter(lambda idx_with_instance: rule.describes(
idx_with_instance[1])),
map(lambda idx_with_instance: idx_with_instance[0]),
list,
)
logger.debug(
f"Depth={depth}, matching_samples={len(matching_sample_indicies)}"
)
if depth >= 1 and len(matching_sample_indicies) >= 1:
if any(
already_covered_indicies.intersection(
set(matching_sample_indicies))):
raise Exception("Rules are overlapping")
already_covered_indicies.update(matching_sample_indicies)
x_train = x[matching_sample_indicies]
y_train = y[matching_sample_indicies]
clf = DecisionTreeClassifier(random_state=self.random_state,
max_depth=depth)
clf.fit(x_train, y_train)
self._clf_by_rule[rule] = clf
not_covered_indices = list(
set(range(len(x))).difference(already_covered_indicies))
default_clf = DecisionTreeClassifier(random_state=self.random_state,
max_depth=self.max_depth)
if len(not_covered_indices) == 0 or self.train_default_on_whole_set:
default_clf.fit(x, y)
else:
x_train = x[not_covered_indices]
y_train = y[not_covered_indices]
default_clf.fit(x_train, y_train)
self._default_clf = default_clf
def predict(self, x):
instance_idx_by_rule = defaultdict(list)
not_covered_indicies = set(range(len(x)))
for rule in self._clf_by_rule.keys():
for idx, single_x in enumerate(x):
if rule.describes(to_instance(single_x)):
instance_idx_by_rule[rule].append(idx)
not_covered_indicies.remove(idx)
label_by_idx = OrderedDict()
for rule, indicies in instance_idx_by_rule.items():
clf = self._clf_by_rule[rule]
x_to_classify = x[indicies]
for label, idx in zip(clf.predict(x_to_classify), indicies):
label_by_idx[idx] = label
x_indicies_not_covered = list(not_covered_indicies)
if len(x_indicies_not_covered) > 0:
x_not_covered_to_classify = x[x_indicies_not_covered]
for label, idx in zip(
self._default_clf.predict(x_not_covered_to_classify),
x_indicies_not_covered,
):
label_by_idx[idx] = label
return list(label_by_idx.values())
def _classify_instance(self, x: List[any]):
clf = self._find_corresponding_classifier(to_instance(x))
return clf.predict([x])[0]
def _find_corresponding_classifier(self, x: Instance) -> any:
matching_rules = pipe(self._clf_by_rule.keys(),
filter(lambda r: r.describes(x)), list)
if len(matching_rules) > 1:
raise Exception("Too many matching rules")
if len(matching_rules) == 0:
return self._default_clf
return self._clf_by_rule[matching_rules[0]]
class ExperimentDataAfterSplit3D:
x_test = attrib()
x_train = attrib()
y_train = attrib()
y_test = attrib()
shape: tuple = attrib()
