class Profile(object):
name = related.StringField(default="__root__")
host = related.StringField(required=False)
schemas = related.ChildField(Namespace, required=False)
globals = related.ChildField(Namespace, required=False)
headers = related.ChildField(Namespace, required=False)
prefixes = related.SequenceField(str, default=None)
extensions = related.SequenceField(str, default=["rigor"])
includes = related.SequenceField(str, default=None)
excludes = related.SequenceField(str, default=None)
concurrency = related.IntegerField(default=5)
retries = related.IntegerField(default=0)
sleep = related.IntegerField(default=60)
retry_failed = related.BooleanField(default=False, required=False)
def __attrs_post_init__(self):
# circumvent frozen error due to immutable
extensions = [
ext[1:] if ext.startswith(".") else ext
for ext in self.extensions or []
]
object.__setattr__(self, "extensions", extensions)
def as_dict(self):
kwargs = related.to_dict(self)
kwargs.pop("profiles", None)
kwargs.pop("file_path", None)
return kwargs
class ExamQuestion:
id = related.StringField(None)
text = related.StringField('')
is_multiple_choice = related.BooleanField(False)
answers = related.SequenceField(ExamQuestionAnswer, None)
points = related.IntegerField(0)
@staticmethod
def from_meta(id: str, meta: ExamQuestionMeta):
meta.validate()
multiple_choice = meta.is_multiple_choice
answers_meta = meta.prepare_answers()
answers = [ExamQuestionAnswer.from_meta(ans) for ans in answers_meta]
return ExamQuestion(id=id,
text=meta.text,
is_multiple_choice=multiple_choice,
answers=answers,
points=meta.points)
def grade(self, meta: ExamQuestionMeta) -> int:
"""
Returns the score for this ExamQuestion instance
"""
grader = get_grader(meta.grader)
if grader:
return grader(meta, self)
return 0
class ModelInfo(Info):
"""Additional information for the model - not applicable to the dataloader
"""
contributors = related.SequenceField(Author, default=[], repr=True, required=False)
cite_as = related.StringField(required=False) # a link or a description how to cite the paper (say a doi link)
trained_on = related.StringField(required=False) # a link or a description of the training dataset
training_procedure = related.StringField(required=False) # brief description about the training procedure for the trained_on dataset.
class ModelDescription(RelatedLoadSaveMixin):
"""Class representation of model.yaml
"""
args = related.ChildField(dict)
info = related.ChildField(ModelInfo)
schema = related.ChildField(ModelSchema)
defined_as = related.StringField(required=False)
type = related.StringField(required=False)
default_dataloader = AnyField(default='.', required=False)
dependencies = related.ChildField(Dependencies,
default=Dependencies(),
required=False)
test = related.ChildField(ModelTest, default=ModelTest(), required=False)
path = related.StringField(required=False)
writers = related.ChildField(dict, default=OrderedDict(), required=False)
# TODO - add after loading validation for the arguments class?
def __attrs_post_init__(self):
if self.defined_as is None and self.type is None:
raise ValueError("Either defined_as or type need to be specified")
self.args = recursive_dict_parse(self.args, 'url',
RemoteFile.from_config)
# parse default_dataloader
if isinstance(self.default_dataloader, dict):
self.default_dataloader = DataLoaderImport.from_config(
self.default_dataloader)
class DocString(object):
value = related.StringField()
content_type = related.StringField(required=False)
line = related.IntegerField(default=6)
@classmethod
def section(cls, title, obj, **kwargs):
size = max(20, len(title))
bar = "=" * size
# if isinstance(obj, dict) and "html" in obj:
# content = str(obj["html"])
# else:
# content = related.to_yaml(obj, **kwargs) if obj else "None"
content = related.to_yaml(obj, **kwargs) if obj else "None"
return "\n".join([bar, str.center(title, size), bar, "", content, ""])
@classmethod
def create(cls, step_result):
return cls(value="\n".join([
cls.section("REQUEST", step_result.fetch),
cls.section("RESPONSE [%s]" %
step_result.status, step_result.response),
cls.section("TRANSFORM", step_result.transform),
cls.section("EXTRACT", step_result.extract),
cls.section("FAILURES",
step_result.failed_validations,
suppress_empty_values=False)
]))
class Element(object):
keyword = related.StringField()
id = related.StringField()
name = related.StringField()
line = related.IntegerField()
description = related.StringField()
type = related.StringField()
steps = related.SequenceField(Step, default=[])
@classmethod
def create(cls, scenario_result):
uuid = "%s;%s" % (urllib.parse.quote_plus(
scenario_result.case.name), scenario_result.uuid)
# scenario step + steps
steps = [Step.create(None, scenario_result)] + \
[Step.create(step_result, scenario_result)
for step_result in scenario_result.step_results]
return cls(
keyword="Scenario",
name=scenario_result.scenario.__name__,
id=uuid,
line=2,
description="",
type="scenario",
steps=steps,
)
class Fetch(object):
url = related.StringField()
method = related.StringField()
kwargs = related.ChildField(dict)
is_form = related.BooleanField()
def get_kwargs(self, is_aiohttp):
kw = self.kwargs.copy()
data = kw.get("data", None)
# aiohttp is different from requests in handling files
# http://aiohttp.readthedocs.io/en/stable/client.html
# http://docs.python-requests.org/en/master/user/quickstart
files = kw.pop("files", None) if is_aiohttp else None
if self.is_form:
if isinstance(data, dict) and isinstance(files, dict):
data.update(files)
else:
kw['data'] = related.to_json(data)
# unlimited timeout if not specified
kw.setdefault("timeout", None)
# add verify = False for requests
if not is_aiohttp:
kw['verify'] = False
return kw
class Feature(object):
uri = related.StringField()
keyword = related.StringField()
id = related.StringField()
name = related.StringField()
line = related.IntegerField()
elements = related.SequenceField(Element, default=[])
description = related.StringField(required=False, default=None)
tags = related.SequenceField(Tag, required=False, default=None)
@classmethod
def create(cls, case_result):
case = case_result.case
uuid = "%s;%s" % (urllib.parse.quote_plus(case.name), case.uuid)
return cls(
uri=case.file_path,
keyword="Feature",
id=uuid,
name=case.name,
line=1,
elements=[
Element.create(scenario_result) for scenario_result in chain(
case_result.passed, case_result.failed)
],
tags=[Tag(name=tag) for tag in case.tags],
)
class Step(object):
keyword = related.StringField()
line = related.IntegerField()
match = related.ChildField(Match)
name = related.StringField()
doc_string = related.ChildField(DocString, required=False, default=None)
result = related.ChildField(StatusResult, required=False, default=None)
@classmethod
def create(cls, step_result, scenario_result):
if step_result is None:
output = related.to_dict(scenario_result.scenario)
output['__file__'] = scenario_result.case.file_path
return cls(keyword="",
line=3,
name="Scenario Setup",
doc_string=DocString.section("SCENARIO", output),
match=Match(),
result=StatusResult.create(True, 0))
else:
return cls(
keyword="",
line=3,
name=step_result.step.description,
doc_string=DocString.create(step_result),
match=Match.create(step_result.step),
result=StatusResult.create(step_result.success,
step_result.duration),
)
class RemoteFile(RelatedConfigMixin):
url = related.StringField()
md5 = related.StringField("", required=False)
name = related.StringField("", required=False)
def __attrs_post_init__(self):
if self.md5 == "":
logger.warning("md5 not specified for url: {}".format(self.url))
if os.path.basename(self.name) != self.name:
logger.warning(
"'name' does not seem to be a valid file name: {}".format(
self.name))
self.name = os.path.basename(self.name)
def validate(self, path):
"""Validate if the path complies with the provided md5 hash
"""
return check_integrity(path, self.md5)
def get_file(self, path):
"""Download the remote file to cache_dir and return
the file path to it
"""
if self.md5:
file_hash = self.md5
else:
file_hash = None
root, filename = os.path.dirname(path), os.path.basename(path)
root = os.path.abspath(root)
download_url(self.url, root, filename, file_hash)
return os.path.join(root, filename)
class Service(object):
name = related.StringField()
image = related.StringField(required=False)
build = related.StringField(required=False)
ports = related.SequenceField(Port, required=False)
volumes = related.SequenceField(str, required=False)
command = related.StringField(required=False)
class RemoteFile(RelatedConfigMixin):
url = related.StringField()
md5 = related.StringField("", required=False)
def __attrs_post_init__(self):
if self.md5 == "":
logger.warn("md5 not specified for url: {}".format(self.url))
def validate(self, path):
"""Validate if the path complies with the provided md5 hash
"""
from kipoi.external.torchvision.dataset_utils import check_integrity
return check_integrity(path, self.md5)
def get_file(self, path):
"""Download the remote file to cache_dir and return
the file path to it
"""
from kipoi.external.torchvision.dataset_utils import download_url
if self.md5:
file_hash = self.md5
else:
file_hash = None
root, filename = os.path.dirname(path), os.path.basename(path)
root = os.path.abspath(root)
download_url(self.url, root, filename, file_hash)
return os.path.join(root, filename)
class DataLoaderDescription(RelatedLoadSaveMixin):
"""Class representation of dataloader.yaml
"""
type = related.StringField()
defined_as = related.StringField()
args = related.MappingField(DataLoaderArgument, "name")
info = related.ChildField(Info)
output_schema = related.ChildField(DataLoaderSchema)
dependencies = related.ChildField(Dependencies,
default=Dependencies(),
required=False)
path = related.StringField(required=False)
postprocessing = related.ChildField(dict,
default=OrderedDict(),
required=False)
def get_example_kwargs(self):
return example_kwargs(self.args)
def print_kwargs(self, format_examples_json=False):
from kipoi.external.related.fields import UNSPECIFIED
if hasattr(self, "args"):
logger.warn(
"No keyword arguments defined for the given dataloader.")
return None
for k in self.args:
print("Keyword argument: `{0}`".format(k))
for elm in ["doc", "type", "optional", "example"]:
if hasattr(self.args[k], elm) and \
(not isinstance(getattr(self.args[k], elm), UNSPECIFIED)):
print(" {0}: {1}".format(elm,
getattr(self.args[k], elm)))
example_kwargs = self.example_kwargs
print("-" * 80)
if hasattr(self, "example_kwargs"):
if format_examples_json:
import json
example_kwargs = json.dumps(example_kwargs)
print("Example keyword arguments are: {0}".format(
str(example_kwargs)))
def __attrs_post_init__(self):
# load additional objects
for k in self.postprocessing:
k_observed = k
if k == 'variant_effects':
k = 'kipoi_veff'
if is_installed(k):
# Load the config properly if the plugin is installed
try:
parser = get_dataloader_yaml_parser(k)
self.postprocessing[k_observed] = parser.from_config(
self.postprocessing[k_observed])
object.__setattr__(self, "postprocessing",
self.postprocessing)
except Exception:
logger.warn(
"Unable to parse {} filed in DataLoaderDescription: {}"
.format(k_observed, self))
class EnvCreateArgs(RelatedConfigMixin):
model = StrSequenceField(str, required=True)
source = related.StringField(required=True)
dataloader = StrSequenceField(str, default=[], required=False)
env = related.StringField(default=None, required=False)
gpu = related.BooleanField(default=False, required=False)
tmpdir = related.StringField(default=None, required=False)
class BubbleChartSettings:
linear_fit = related.BooleanField(False, key='linearFit')
result_limit = related.IntegerField(100, key='resultLimit')
show_legend = related.BooleanField(False, key='showLegend')
x_axis = related.StringField('', key='xAxis')
y_axis = related.StringField('', key='yAxis')
z_axis = related.StringField('', key='zAxis')
class MetadataStruct(RelatedConfigMixin):
doc = related.StringField()
type = related.ChildField(MetadataType, required=False)
name = related.StringField(required=False)
def compatible_with_batch(self, batch, verbose=True):
"""Checks compatibility with a particular numpy array
Args:
batch: numpy array of a batch
verbose: print the fail reason
"""
def print_msg(msg):
if verbose:
print("MetadataStruct mismatch")
print(msg)
# custom classess
if self.type == MetadataType.GENOMIC_RANGES:
if not isinstance(batch, GenomicRanges):
# TODO - do we strictly require the GenomicRanges class?
# - relates to metadata.py TODO about numpy_collate
# for now we should just be able to convert to the GenomicRanges class
# without any errors
try:
GenomicRanges.from_dict(batch)
except Exception as e:
print_msg("expecting a GenomicRanges object or a GenomicRanges-like dict")
print_msg("convertion error: {0}".format(e))
return False
else:
return True
else:
return True
# type = np.ndarray
if not isinstance(batch, np.ndarray):
print_msg("Expecting a np.ndarray. Got type(batch) = {0}".format(type(batch)))
return False
if not batch.ndim >= 1:
print_msg("The array is a scalar (expecting at least the batch dimension)")
return False
bshape = batch.shape[1:]
# scalars
if self.type in {MetadataType.INT, MetadataType.STR, MetadataType.FLOAT}:
if bshape != () and bshape != (1,):
print_msg("expecting a scalar, got an array with shape (without the batch axis): {0}".format(bshape))
return False
# arrays
# - no checks
return True
class Info(object):
"""General information about the API."""
version = related.StringField(required=True)
title = related.StringField(required=True)
description = related.StringField(required=False)
termsOfService = related.StringField(required=False)
contact = related.ChildField(Contact, required=False)
license = related.ChildField(License, required=False)
class MyModel(object):
is_for = related.StringField(key="for")
criss = related.StringField(key="cross")
cross = related.StringField(key="criss")
is_not = related.BooleanField(key="not")
is_list = related.SequenceField(str, key="list")
is_type = related.ChildField(DataType, key="type")
is_dict = related.MappingField(MyChild, "int", key="dict", required=False)
class AlertNotification(AlertNotificationMeta):
'''Object representation of a alert notification.
'''
query_interval = related.StringField(key='queryInterval', required=False)
reported_val = related.StringField(key='reportedVal', required=False)
message = related.StringField(key='message', required=False)
dimension_val = related.StringField(key='dimensionVal', required=False)
class Requestor(object):
path = related.StringField()
method = related.ChildField(Method, default=Method.GET)
host = related.StringField(required=False)
headers = related.ChildField(Namespace, required=False)
params = related.ChildField(Namespace, required=False)
data = related.ChildField(object, required=False)
form = related.ChildField(Namespace, required=False)
files = related.ChildField(Namespace, required=False)
status = related.SequenceField(int, required=False)
def get_params(self, namespace):
dd = self.params.evaluate(namespace) if self.params else {}
params = []
for key, value in dd.items():
if isinstance(value, (tuple, list, set)):
for item in value:
params.append((key, str(item)))
else:
params.append((key, str(value)))
return params
def get_form(self, namespace):
return self.form.evaluate(namespace) if self.form else {}
def get_files(self, dir_path, namespace):
files = self.files.evaluate(namespace) if self.files else {}
files = {
k: open(os.path.join(dir_path, v), "rb")
for k, v in files.items()
}
return files
def get_body(self, namespace):
get_logger().debug("enter get_body",
data_type=type(self.data),
data=self.data)
body = None
if isinstance(self.data, str):
body = Namespace.render(self.data, namespace)
if isinstance(body, Namespace):
body = body.evaluate(namespace)
elif isinstance(self.data, dict):
body = Namespace(self.data).evaluate(namespace)
get_logger().debug("render get_body", body_type=type(body), body=body)
return body
def get_data(self, namespace):
""" Returns body or form and a flag indicating if a form or not."""
body = self.get_body(namespace) if self.data else None
form = self.get_form(namespace) if self.form else None
return (body, False) if body else (form, True)
class AlertDefinition(AlertDefinitionMeta):
'''Object representation of a alert definition.
'''
# TODO(toshi): Perhaps this should have the basic _uri be part of this.
checks = related.SequenceField(AlertCheck)
time_granularity = related.StringField(key='timeGranularity')
field_id = related.StringField(key='fieldId')
dimension_name = related.StringField(key='dimensionName')
class AlertDefinitionMeta(object):
'''Object representation of the metadata for an Alert Definition.
'''
# NOTE(toshi): Maybe we want to convert this to user?
user_id = related.StringField(key='userId')
# $uri or alert definition id
# id = related.IntegerField()
uri = related.StringField(key='$uri')
class QueryDefinition:
id = related.StringField()
advanced_fields = related.SequenceField(AdvancedFieldDefinition, [],
key='advancedFields')
advanced_filters = related.SequenceField(dict, [], key='advancedFilters')
advanced_groups = related.SequenceField(dict, [], key='advancedGroups')
name = related.StringField(required=False)
magic_filter_ids = related.SequenceField(str, [], key='magicFilters')
date_range_id = related.StringField('', key='dateRangeId')
group_by = related.StringField('', key='groupBy')
class AlertNotificationMeta(object):
'''Object representation of the metadata for an Alert Notification.
'''
generation_date = related.StringField(key='generationDate', required=False)
# TODO(toshi): Possible to change constructor to take in just a string
alert_definition = related.ChildField(RefObject,
key='alertDefinition',
required=False)
uri = related.StringField(key='$uri', required=False)
class AnimatedMapSettings:
base_layer = related.ChildField(BaseMapLayer, BaseMapLayer.STREETS, key='baseLayer')
current_display = related.StringField('dots', key='currentDisplay')
map_center = related.ChildField(Coordinates, [0.0, 0.0], key='mapCenter')
overlay_layers = related.SequenceField(
str, DEFAULT_OVERLAY_LAYERS, key='overlayLayers'
)
selected_field = related.StringField('', key='selectedField')
selected_geo_tiles = related.StringField('', key='selectedGeoTiles')
zoom_level = related.FloatField(1.0, key='zoomLevel')
class Step(object):
description = related.StringField()
request = related.ChildField(Requestor)
extract = related.ChildField(Namespace, default=Namespace())
iterate = related.ChildField(Iterator, default=Iterator())
validate = related.SequenceField(Validator, required=False)
condition = related.BooleanField(required=False, default=None)
transform = related.StringField(required=False, default=None)
name = related.StringField(required=False, default=None)
sleep = related.FloatField(required=False, default=0.01)
class HeatTileSettings:
divergent_coloration = related.BooleanField(True, key='divergentColoration')
first_y_axis_selections = related.SequenceField(str, [], key='firstYaxisSelections')
invert_coloration = related.BooleanField(False, key='invertColoration')
log_scaling = related.BooleanField(True, key='logScaling')
result_limit = related.IntegerField(100, key='resultLimit')
selected_field = related.StringField('', key='selectedField')
show_time_on_y_axis = related.BooleanField(True, key='showTimeOnYAxis')
sort_order = related.ChildField(SortOrder, SortOrder.DESCENDING, key='sortOrder')
sort_on = related.StringField('', key='sortOn')
use_ethiopian_dates = related.BooleanField(False, key='useEthiopianDates')
class AdvancedFieldDefinition:
id = related.StringField()
calculation = related.ChildField(dict)
canonical_name = related.StringField(key='canonicalName')
category = related.ChildField(dict)
customizable_filter_items = related.SequenceField(
dict, key='customizableFilterItems')
description = related.StringField()
label = related.StringField()
short_name = related.StringField(key='shortName')
source = related.ChildField(dict)
class Usuario:
Username = related.StringField(required=False)
Nombre = related.StringField(required=False)
Apellido = related.StringField(required=False)
@staticmethod
def load(dictionary):
return to_model(Usuario, dictionary)
def provide(self):
d = to_dict(self)
return d
class LegendSettings(object):
legend_font_color = related.StringField(FONT_COLOR_DEFAULT,
key='legendFontColor')
legend_font_family = related.StringField(FONT_FAMILY_DEFAULT,
key='legendFontFamily')
legend_font_size = related.StringField(FONT_16, key='legendFontSize')
legend_placement = related.ChildField(LegendPlacement,
LegendPlacement.BOTTOM,
key='legendPlacement')
overlap_legend_with_chart = related.BooleanField(
False, key='overlapLegendWithChart')
show_legend = related.BooleanField(True, key='showLegend')
