class SourceConfig(RelatedLoadSaveMixin):
test = related.ChildField(TestConfig, required=False)
# default dependencies
dependencies = related.ChildField(Dependencies,
default=Dependencies(),
required=False)
path = related.StringField(required=False)
class VisualizationSettings(object):
id = related.StringField()
title_settings = related.ChildField(TitleSettings, key='titleSettings')
group_by_settings = related.ChildField(GroupBySettings,
key='groupBySettings')
view_type_settings = related.ChildField(ViewTypeSettings,
key='viewTypeSettings')
class AsyncSession(Session):
loop = related.ChildField(object)
connector = related.ChildField(TCPConnector)
http = related.ChildField(object)
def run(self):
future = asyncio.ensure_future(self.run_suite())
self.loop.run_until_complete(future)
results = future.result()
self.http.close()
return results
async def run_suite(self):
tasks = []
for case, scenario in self.case_scenarios():
tasks.append(
asyncio.ensure_future(self.run_case_scenario(case, scenario)))
return await asyncio.gather(*tasks, return_exceptions=False)
async def run_case_scenario(self, case, scenario):
from . import State
with State(session=self, case=case, scenario=scenario) as state:
async for step_result in self.iter_steps(state):
state.add_step(step_result)
return state.result()
class ScenarioResult(object):
uuid = related.UUIDField()
case = related.ChildField(Case, required=None)
scenario = related.ChildField(Namespace, required=None)
success = related.BooleanField(default=True)
step_results = related.SequenceField(StepResult, default=[])
suite = related.ChildField(Suite, required=False)
class Step(object):
keyword = related.StringField()
line = related.IntegerField()
match = related.ChildField(Match)
name = related.StringField()
doc_string = related.ChildField(DocString, required=False, default=None)
result = related.ChildField(StatusResult, required=False, default=None)
@classmethod
def create(cls, step_result, scenario_result):
if step_result is None:
output = related.to_dict(scenario_result.scenario)
output['__file__'] = scenario_result.case.file_path
return cls(keyword="",
line=3,
name="Scenario Setup",
doc_string=DocString.section("SCENARIO", output),
match=Match(),
result=StatusResult.create(True, 0))
else:
return cls(
keyword="",
line=3,
name=step_result.step.description,
doc_string=DocString.create(step_result),
match=Match.create(step_result.step),
result=StatusResult.create(step_result.success,
step_result.duration),
)
class DataLoaderImport(RelatedConfigMixin):
"""Dataloader specification for the import
"""
defined_as = related.StringField()
default_args = related.ChildField(dict, default=OrderedDict(), required=False)
# specify also dataloader dependencies explicitly
dependencies = related.ChildField(Dependencies,
default=Dependencies(),
required=False)
# whether to parse the dependencies from the dataloader when installing it
parse_dependencies = related.BooleanField(default=True, required=False)
def get(self):
"""Get the dataloader
"""
from kipoi.data import BaseDataLoader
from copy import deepcopy
obj = load_obj(self.defined_as)
# check that it inherits from BaseDataLoader
if not inherits_from(obj, BaseDataLoader):
raise ValueError("Dataloader: {} doen't inherit from kipoi.data.BaseDataLoader".format(self.defined_as))
# override the default arguments
if self.default_args:
obj = override_default_kwargs(obj, self.default_args)
# override also the values in the example in case
# they were previously specified
for k, v in six.iteritems(self.default_args):
if not isinstance(obj.args[k].example, UNSPECIFIED):
obj.args[k].example = v
return obj
class MapSettings:
base_layer = related.ChildField(BaseMapLayer, BaseMapLayer.STREETS, key='baseLayer')
current_display = related.StringField('dots', key='currentDisplay')
map_center = related.ChildField(Coordinates, [0.0, 0.0], key='mapCenter')
overlay_layers = related.SequenceField(
str, DEFAULT_OVERLAY_LAYERS, key='overlayLayers'
)
selected_field = related.StringField('', key='selectedField')
selected_geo_tiles = related.StringField('', key='selectedGeoTiles')
zoom_level = related.FloatField(1.0, key='zoomLevel')
show_administrative_boundaries = related.BooleanField(
True, key='showAdminBoundaries'
)
show_labels = related.BooleanField(False, key='showLabels')
fill_opacity = related.FloatField(0.8, key='fillOpacity')
tooltip_font_color = related.StringField(FONT_COLOR_DEFAULT, key='tooltipFontColor')
tooltip_font_family = related.StringField(
FONT_FAMILY_DEFAULT, key='tooltipFontFamily'
)
tooltip_font_size = related.StringField(FONT_12, key='tooltipFontSize')
tooltip_bold = related.BooleanField(False, key='tooltipBold')
# HACK(stephen, moriah): This setting is really poorly designed. It encodes
# a color value + alpha as a dict instead of rgba or hex + alpha which can
# be represented as a string. FIX THIS OR REMOVE IT.
tooltip_background_color = related.ChildField(
dict, DEFAULT_MAP_TOOLTIP_BACKGROUND, key='tooltipBackgroundColor'
)
class AsyncSession(Session):
loop = related.ChildField(object)
connector = related.ChildField(TCPConnector)
http = related.ChildField(object)
def run(self, case_scenarios=None):
# run and get results
future = asyncio.ensure_future(self.run_suite(case_scenarios))
self.loop.run_until_complete(future)
results = future.result()
# close http
future = asyncio.ensure_future(self.close_http())
self.loop.run_until_complete(future)
return results
async def close_http(self):
await self.http.close()
async def run_suite(self, case_scenarios=None):
tasks = []
if case_scenarios is None:
case_scenarios = self.case_scenarios()
for case, scenario in case_scenarios:
tasks.append(
asyncio.ensure_future(self.run_case_scenario(case, scenario))
)
return await asyncio.gather(*tasks, return_exceptions=False)
async def run_case_scenario(self, case, scenario):
# Allows only 1 scenario run at a time with same semaphore name
if case.semaphore is not None:
async with self.suite.semaphores[case.semaphore]:
return await self.run_single_case_scenario(case, scenario)
class UtilConfig(object):
lang1 = related.ChildField(LanguageInfo)
lang2 = related.ChildField(LanguageInfo)
redo = related.BooleanField(default=False)
path = related.StringField(default=".")
num = related.IntegerField(default=10)
record = related.BooleanField(default=True)
class CommandConfig(object):
target_device_list = related.SequenceField(TargetDevice)
database = related.ChildField(Database, required=False)
influxdb = related.ChildField(Influxdb, required=False)
set_measurement_interval = related.IntegerField(required=False)
dry_run = related.BooleanField(default=False, required=False)
no_scan = related.BooleanField(default=False, required=False)
class DataLoaderDescription(RelatedLoadSaveMixin):
"""Class representation of dataloader.yaml
"""
type = related.StringField()
defined_as = related.StringField()
args = related.MappingField(DataLoaderArgument, "name")
info = related.ChildField(Info)
output_schema = related.ChildField(DataLoaderSchema)
dependencies = related.ChildField(Dependencies,
default=Dependencies(),
required=False)
path = related.StringField(required=False)
postprocessing = related.ChildField(dict,
default=OrderedDict(),
required=False)
def get_example_kwargs(self):
return example_kwargs(self.args)
def print_kwargs(self, format_examples_json=False):
from kipoi.external.related.fields import UNSPECIFIED
if hasattr(self, "args"):
logger.warn(
"No keyword arguments defined for the given dataloader.")
return None
for k in self.args:
print("Keyword argument: `{0}`".format(k))
for elm in ["doc", "type", "optional", "example"]:
if hasattr(self.args[k], elm) and \
(not isinstance(getattr(self.args[k], elm), UNSPECIFIED)):
print(" {0}: {1}".format(elm,
getattr(self.args[k], elm)))
example_kwargs = self.example_kwargs
print("-" * 80)
if hasattr(self, "example_kwargs"):
if format_examples_json:
import json
example_kwargs = json.dumps(example_kwargs)
print("Example keyword arguments are: {0}".format(
str(example_kwargs)))
def __attrs_post_init__(self):
# load additional objects
for k in self.postprocessing:
k_observed = k
if k == 'variant_effects':
k = 'kipoi_veff'
if is_installed(k):
# Load the config properly if the plugin is installed
try:
parser = get_dataloader_yaml_parser(k)
self.postprocessing[k_observed] = parser.from_config(
self.postprocessing[k_observed])
object.__setattr__(self, "postprocessing",
self.postprocessing)
except Exception:
logger.warn(
"Unable to parse {} filed in DataLoaderDescription: {}"
.format(k_observed, self))
class Profile(object):
name = related.StringField(default="__root__")
host = related.StringField(required=False)
schemas = related.ChildField(Namespace, required=False)
globals = related.ChildField(Namespace, required=False)
headers = related.ChildField(Namespace, required=False)
prefixes = related.SequenceField(str, default=None)
extensions = related.SequenceField(str, default=["rigor"])
includes = related.SequenceField(str, default=None)
excludes = related.SequenceField(str, default=None)
concurrency = related.IntegerField(default=5)
retries = related.IntegerField(default=0)
sleep = related.IntegerField(default=60)
retry_failed = related.BooleanField(default=False, required=False)
def __attrs_post_init__(self):
# circumvent frozen error due to immutable
extensions = [
ext[1:] if ext.startswith(".") else ext
for ext in self.extensions or []
]
object.__setattr__(self, "extensions", extensions)
def as_dict(self):
kwargs = related.to_dict(self)
kwargs.pop("profiles", None)
kwargs.pop("file_path", None)
return kwargs
class BoxPlot(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(BoxPlotSettings,
BoxPlotSettings(),
key='viewSpecificSettings')
view_type = related.StringField('BOX', key='viewType')
class Definition(object):
key = related.StringField(required=True)
type = related.ChildField(DataType, required=False)
required = related.SequenceField(str, required=False)
allOf = related.SequenceField(object, required=False)
anyOf = related.SequenceField(object, required=False)
is_not = related.SequenceField(object, required=False, key="not")
properties = related.ChildField(object, required=False)
class HeatMap(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(HeatMapSettings,
HeatMapSettings(),
key='viewSpecificSettings')
view_type = related.StringField('HEATMAP', key='viewType')
class ExpandoTree(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(ExpandoTreeSettings,
ExpandoTreeSettings(),
key='viewSpecificSettings')
view_type = related.StringField('EXPANDOTREE', key='viewType')
class Info(object):
"""General information about the API."""
version = related.StringField(required=True)
title = related.StringField(required=True)
description = related.StringField(required=False)
termsOfService = related.StringField(required=False)
contact = related.ChildField(Contact, required=False)
license = related.ChildField(License, required=False)
class BumpChart(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(BumpChartSettings,
BumpChartSettings(),
key='viewSpecificSettings')
view_type = related.StringField('BUMP_CHART', key='viewType')
class ConstantThresholdConfig:
hyperparams = related.ChildField(ConstantThresholdHyperparameters)
training_meta_data = related.ChildField(
ConstantThresholdTrainingMetaData,
key="trainingMetaData",
default=ConstantThresholdTrainingMetaData(),
)
params = related.ChildField(ConstantThresholdParams, required=False)
class Table(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(TableSettings,
TableSettings(),
key='viewSpecificSettings')
view_type = related.StringField('TABLE', key='viewType')
class Sunburst(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(SunburstSettings,
SunburstSettings(),
key='viewSpecificSettings')
view_type = related.StringField('SUNBURST', key='viewType')
class MyModel(object):
is_for = related.StringField(key="for")
criss = related.StringField(key="cross")
cross = related.StringField(key="criss")
is_not = related.BooleanField(key="not")
is_list = related.SequenceField(str, key="list")
is_type = related.ChildField(DataType, key="type")
is_dict = related.MappingField(MyChild, "int", key="dict", required=False)
is_enum = related.ChildField(IntEnum, key="enum", required=False)
class CaseResult(object):
suite = related.ChildField(Suite)
case = related.ChildField(Case)
passed = related.SequenceField(ScenarioResult, default=[])
failed = related.SequenceField(ScenarioResult, default=[])
@property
def success(self):
return bool(self.passed and not self.failed)
class Step(object):
description = related.StringField()
request = related.ChildField(Requestor)
extract = related.ChildField(Namespace, default=Namespace())
iterate = related.ChildField(Iterator, default=Iterator())
validate = related.SequenceField(Validator, required=False)
condition = related.BooleanField(required=False, default=None)
transform = related.StringField(required=False, default=None)
name = related.StringField(required=False, default=None)
sleep = related.FloatField(required=False, default=0.01)
class HParams(RelatedLoadSaveMixin):
"""Hyper-nparameters yaml file
"""
model = related.ChildField(ModelHParams, ModelHParams(), required=False)
data = related.ChildField(DataHParams, DataHParams(), required=False)
train = related.ChildField(TrainHParams, TrainHParams(), required=False)
evaluate = related.ChildField(EvalHParams, EvalHParams(), required=False)
modisco = related.ChildField(dict, default={}, required=False)
# modisco = related.ChildField(ModiscoHParams, ModiscoHParams(), required=False)
path = related.StringField(required=False)
class AnimatedMapSettings:
base_layer = related.ChildField(BaseMapLayer, BaseMapLayer.STREETS, key='baseLayer')
current_display = related.StringField('dots', key='currentDisplay')
map_center = related.ChildField(Coordinates, [0.0, 0.0], key='mapCenter')
overlay_layers = related.SequenceField(
str, DEFAULT_OVERLAY_LAYERS, key='overlayLayers'
)
selected_field = related.StringField('', key='selectedField')
selected_geo_tiles = related.StringField('', key='selectedGeoTiles')
zoom_level = related.FloatField(1.0, key='zoomLevel')
class AxesSettings(object):
x_axis_settings = related.ChildField(XAxisSettings,
XAxisSettings(),
key='xAxis')
y1_axis_settings = related.ChildField(YAxisSettings,
YAxisSettings(),
key='y1Axis')
y2_axis_settings = related.ChildField(YAxisSettings,
YAxisSettings(),
key='y2Axis')
class BubbleChart(object):
axes_settings = related.ChildField(AxesSettings,
AxesSettings(),
key='axesSettings')
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(BubbleChartSettings,
BubbleChartSettings(),
key='viewSpecificSettings')
view_type = related.StringField('BUBBLE_CHART', key='viewType')
class Map(object):
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
legend_settings = related.ChildField(LegendSettings,
LegendSettings(),
key='legendSettings')
view_specific_settings = related.ChildField(MapSettings,
MapSettings(),
key='viewSpecificSettings')
view_type = related.StringField('MAP', key='viewType')
class HeatTile(object):
axes_settings = related.ChildField(AxesSettings,
AxesSettings(),
key='axesSettings')
series_settings = related.ChildField(SeriesSettings,
SeriesSettings(),
key='seriesSettings')
view_specific_settings = related.ChildField(HeatTileSettings,
HeatTileSettings(),
key='viewSpecificSettings')
view_type = related.StringField('HEATTILES', key='viewType')
