def load_scheme(yaml_text):
'''
Some optional keys have enforced default values, otherwise use dict.get()
'''
schema = MapPattern(
Str(),
Map({
'description':
Str(),
Optional('alias'):
Str(),
'version':
Str(),
'directives':
MapPattern(
Str(),
Map({
Optional('description'):
Str(),
'formats':
Seq(Enum(FORMATS)),
Optional('prepare'):
Map({
Optional('remove_whitespace', default=False): Bool(),
Optional('remove_characters', default=['']):
Seq(Str()),
Optional('strip_characters', default=['']): Seq(Str()),
}),
Optional('validate'):
Map({
Optional('alphabet'): Enum(ALPHABETS),
Optional('min_length'): Int(),
Optional('max_length'): Int(),
}),
Optional('target'):
Str(),
Optional('helper', default=False):
Bool()
}),
),
'algorithm':
Enum(ALGORITHMS),
'encodings':
MapPattern(
Str(),
Map({
'type':
Enum(ENCODINGS),
Optional('length', default=0):
Int(),
Optional('prefix', default=''):
Str(),
Optional('separator'):
Map({
'character': Str(),
'interval': Int()
})
}))
}))
return load(yaml_text, schema)
def _get_info(cls):
# https://hitchdev.com/strictyaml
schema = Map({
Optional("knobs"):
Map({
Optional("log_to_console"): Bool(),
Optional("log_level_debug"): Bool(),
})
| EmptyDict(),
"globals":
Map({
"topic_prefix":
Str(),
Optional(
"reconnect_interval",
default=const.MQTT_DEFAULT_RECONNECT_INTERVAL,
):
Float(),
Optional("poll_interval",
default=const.MYQ_DEFAULT_POLL_INTERVAL):
Float(),
Optional(
"periodic_mqtt_report",
default=const.DEFAULT_PERIODIC_MQTT_REPORT,
):
Float(),
Optional("user_agent"):
Str(),
}),
"mqtt":
Map({
"host":
Str(),
Optional("client_id", default=const.MQTT_DEFAULT_CLIENT_ID):
Str(),
Optional("username"):
Str(),
Optional("password"):
Str(),
}),
"myq":
Map({
"email": Email(),
"password": Str(),
}),
Optional("alias"):
MapPattern(Str(), Str()) | EmptyDict(),
})
if not cls._info:
config_filename = cls._get_config_filename()
logger.info("loading yaml config file %s", config_filename)
with open(config_filename, "r") as ymlfile:
raw_cfg = load(ymlfile.read(), schema).data
cls._parse_raw_cfg(raw_cfg)
return cls._info
def __init__(self):
super().__init__({
Optional(conditions.ALL_OF): self,
Optional(conditions.ENDS_EARLIER_THAN): Float(),
Optional(conditions.ENDS_LATER_THAN): Float(),
Optional(conditions.NONE_OF): self,
Optional(conditions.ONE_OF): self,
Optional(conditions.STARTS_EARLIER_THAN): Float(),
Optional(conditions.STARTS_LATER_THAN): Float(),
Optional(conditions.WEEKDAY): Bool(),
Optional(conditions.WEEKEND): Bool(),
})
def schema(cls):
basic_schema = {
# `name` is not part of the sub schema, name is the title of the block.
# `default` must be provided in the sub schemas
"label": Str(),
"type": Str(),
Optional("variable"): Str(),
Optional("helptext"): Str(),
Optional("visibility"): Int(),
Optional("required"): Bool(),
Optional("readonly"): Bool(),
}
return Map(basic_schema)
def get_world_schema():
"""
Getter for world schema
:return: schema that is used to verify the world yaml
"""
return Map({
'robots':
Seq(
Map({
'name':
Str(),
'center_x':
Int(),
'center_y':
Int(),
'orientation':
Int(),
Optional('type'):
Str(),
Optional('parts'):
Seq(ConfigChecker.get_robot_part_schema())
})),
'board_height':
Int(),
'board_width':
Int(),
'board_color':
CommaSeparated(Int()),
'obstacles':
Seq(
Map({
'name': Str(),
'type': Str(),
Optional('outer_spacing'): Int(),
Optional('depth'): Int(),
Optional('color'): CommaSeparated(Int()),
Optional('border_width'): Int(),
Optional('inner_radius'): Int(),
Optional('x'): Int(),
Optional('y'): Int(),
Optional('width'): Int(),
Optional('height'): Int(),
Optional('angle'): Int(),
Optional('movable'): Bool(),
Optional('hole'): Bool(),
Optional('radius'): Int(),
})),
})
def _get_base_part_mapping(cls) -> dict:
"""Base mapping for segment/phase schemas."""
polar_coeff_schema = CommaSeparated(Float()) | Str()
polar_schema = Map({
"CL": polar_coeff_schema,
"CD": polar_coeff_schema
}) | Str()
return {
# TODO: this mapping covers all possible segments, but some options are relevant
# only for some segments. A better check could be done in second-pass validation.
Optional("target", default=None):
cls._get_target_schema(),
Optional("engine_setting", default=None):
cls._get_value_schema(Str(), False),
Optional(POLAR_TAG, default=None):
polar_schema,
Optional("thrust_rate", default=None):
cls._get_value_schema(has_unit=False),
Optional("climb_thrust_rate", default=None):
cls._get_value_schema(has_unit=False),
Optional("time_step", default=None):
cls._get_value_schema(),
Optional("maximum_flight_level", default=None):
cls._get_value_schema(has_unit=False),
Optional("mass_ratio", default=None):
cls._get_value_schema(has_unit=False),
Optional("reserve_mass_ratio", default=None):
cls._get_value_schema(has_unit=False),
Optional("use_max_lift_drag_ratio", default=None):
cls._get_value_schema(Bool(), False),
}
def _personal_settings():
settings_file = DIR.key.joinpath("personalsettings.yml")
if not settings_file.exists():
settings_file.write_text(("engine:\n"
" rewrite: no\n"
" cprofile: no\n"
"params:\n"
" python version: 3.5.0\n"))
return load(
settings_file.bytes().decode("utf8"),
Map({
"engine": Map({
"rewrite": Bool(),
"cprofile": Bool()
}),
"params": Map({"python version": Str()}),
}),
)
def load_data(filename):
schema = MapPattern(
Regex(u'[A-Za-z. ]+'),
MapPattern(
Regex(u'[A-Za-z\\-. ]+'),
Map({
'Description': Regex('.+'),
'Free': Bool(),
'Link': Url(),
}, Regex(u'.+'))))
with open(filename) as f:
return strictyaml.load(f.read(), schema).data
class Config:
SCHEMA = Map(
dict(image_scale=Float(),
frame_limit=Int(),
dump_stats=Bool(),
alignment=Map(
dict(enabled=Bool(),
mode=Str(),
max_iterations=Int(),
termination_eps=Float()))))
def __init__(self, file_or_yaml=open('config.yaml', 'r')):
if isinstance(file_or_yaml, str):
yaml_str = file_or_yaml
else:
yaml_str = file_or_yaml.read()
file_or_yaml.close()
self.cfg = load(yaml_str, Config.SCHEMA).data
def get(self, key):
return self.cfg[key]
def is_pipelines_config_valid(strictyaml_pipelines: YAML) -> YAML:
"""
TODO: Refactor to test and analyzer specific config validation.
"""
pipelines_schema = Map({
"pipelines":
Seq(
Map({
"name":
Str(),
"type":
Enum(["test", "analyzer"]),
Optional("coverage"):
Str(),
Optional("commands"):
Map({
"partial-scope": Str(),
"full-scope": Str()
}),
Optional("dirs"):
Seq(
Map({
"path": Str(),
Optional("full-scope", default=False): Bool()
})),
Optional("files"):
Seq(
Map({
"path": Str(),
Optional("full-scope", default=False): Bool()
}))
}))
})
try:
strictyaml_pipelines.revalidate(pipelines_schema)
return True
except YAMLValidationError:
return False
def config() -> Config:
CONFIG_DIR.mkdir(parents=True, exist_ok=True)
config_file = CONFIG_DIR / 'config.yaml'
if not config_file.exists():
sys.exit(f'{config_file} not found')
restaurant_schema = Map({
'url': Str(),
'name': Str(),
})
schema = Map({
'restaurants':
Seq(restaurant_schema),
'mail':
Map({
'enable': Bool(),
'sender': Str(),
'password': Str(),
'smtp_server': Str(),
'recipients': UniqueSeq(Str()),
}),
})
yaml = load(config_file.read_text(), schema)
_config = Config()
for i in yaml.get('restaurants', []):
restaurant = Restaurant(i['url'], i['name'])
_config.restaurants.append(restaurant)
mail_config = Mail(
enable=yaml['mail']['enable'],
sender=yaml['mail']['sender'],
password=yaml['mail']['password'],
smtp_server=yaml['mail']['smtp_server'],
recipients=yaml['mail']['recipients'],
)
_config.mail = mail_config
return _config
def build_schema_for_regs():
"""
Returns
-------
strictyaml.Map
schema used for validation and type-coercion
"""
schemas = {}
for elem in artm.regularizers.__all__:
if "Regularizer" in elem:
class_of_object = getattr(artm.regularizers, elem)
res = build_schema_from_signature(class_of_object)
if elem in ["SmoothSparseThetaRegularizer", "SmoothSparsePhiRegularizer",
"DecorrelatorPhiRegularizer"]:
res[Optional("relative", default=None)] = Bool()
res = wrap_in_map(res)
specific_schema = Map({class_of_object.__name__: res})
schemas[class_of_object.__name__] = specific_schema
return schemas
def choose_validator(param):
"""
Parameters
----------
param : inspect.Parameter
Returns
-------
instance of strictyaml.Validator
"""
if param.annotation is int:
return Int()
if param.annotation is float:
return Float()
if param.annotation is bool:
return Bool()
if param.annotation is str:
return Str()
if param.name in ARTM_TYPES:
return ARTM_TYPES[param.name]
return Any()
class WooSchema:
"""Schema for localization YAML files."""
# https://github.com/woocart/woocart-defaults/blob/master/src/importers/class-woopage.php#L14
productMeta = {
"title": Str(),
"description": Str(),
Optional("price"): Str(),
Optional("category"): Str(),
"images": Seq(Str()),
}
# https://github.com/woocart/woocart-defaults/blob/master/src/importers/class-woopage.php#L14
pageMeta = {
"post_title": Str(),
Optional("post_name"): Str(),
Optional("post_excerpt"): Str(),
"post_status": Enum(["draft", "publish"]),
"post_type": Enum(["page", "post"]),
Optional("post_category"): Str(),
Optional("meta_input"): MapPattern(Str(), Str()),
Optional("woocart_defaults"): MapPattern(Str(), Str()),
}
localization = {
"woo/woocommerce_default_country":
Enum(COUNTRIES),
"wp/date_format":
Enum(["d/m/Y", "Y-m-d", "F j, Y", "m/d/Y"]),
"wp/time_format":
Enum(["H:i", "g:i A"]),
"wp/start_of_week":
Enum(["1", "2", "3", "4", "5", "6", "7"]),
"wp/timezone_string":
Enum(TIMEZONES),
"wp/blog_charset":
Enum(["UTF-8"]),
"wp/DEFAULT_WPLANG":
Enum(WPLANGS),
Optional("wp/blogdescription"):
Str(),
Optional("wp/woocommerce_demo_store_notice"):
Str(),
"woo/woocommerce_weight_unit":
Enum(["kg", "k", "lbs", "oz"]),
"woo/woocommerce_dimension_unit":
Enum(["m", "cm", "mm", "in", "yd"]),
"woo/woocommerce_currency":
Enum(CURRENCIES),
"woo/woocommerce_currency_pos":
Enum(["right_space", "left_space", "left", "right"]),
"woo/woocommerce_price_thousand_sep":
Enum([".", ","]),
"woo/woocommerce_price_decimal_sep":
Enum([",", "."]),
"woo/woocommerce_price_num_decimals":
Enum(["2"]),
Optional("woo/woocommerce_tax_classes"):
Seq(Str()),
"woo/woocommerce_bacs_settings":
Map({
"enabled": Bool(),
Optional("title"): Str(),
Optional("description"): Str(),
Optional("instructions"): Str(),
Optional("account_name"): Str(),
Optional("account_number"): Str(),
Optional("sort_code"): Str(),
Optional("bank_name"): Str(),
Optional("iban"): Str(),
Optional("bic"): Str(),
Optional("account_details"): Str(),
}),
"woo/woocommerce_cod_settings":
Map({
"enabled": Bool(),
Optional("title"): Str(),
Optional("description"): Str(),
Optional("instructions"): Str(),
Optional("enable_for_methods"): Str(),
Optional("enable_for_virtual"): Bool(),
}),
"woo/woocommerce_checkout_privacy_policy_text":
Str(),
"woo/woocommerce_registration_privacy_policy_text":
Str(),
".woo/woocommerce_bacs_settings_format":
Enum(["serialized"]),
".woo/woocommerce_cod_settings_format":
Enum(["serialized"]),
Optional(".woo/woocommerce_tax_classes_format"):
Enum(["implode_newline"]),
}
@staticmethod
def load(path: Path, schema_pointer):
"""Load and validate .yaml file."""
schema = copy.deepcopy(schema_pointer)
with path.open() as f:
yaml = f.read()
data = yaml_load(yaml, Any())
is_template = path.name == "template.yaml"
# Replace real Country and Timezone values with fakes
if is_template:
schema["woo/woocommerce_default_country"] = Enum(["LL"])
schema["wp/timezone_string"] = Enum(["Region/Country"])
schema["wp/DEFAULT_WPLANG"] = Enum(["ll_LL"])
schema["woo/woocommerce_currency"] = Enum(["LLL"])
if "woo/woocommerce_tax_classes" in data:
# Inspect that tax classes and taxes match
# create enum for taxes from defined tax_classes
tax_classes = [
str(tax).lower().replace(" ", "-")
for tax in data["woo/woocommerce_tax_classes"]
]
# +1 is for standard schema which is never defined in tax class
for x in range(len(tax_classes) + 1):
# start counting with 1
schema[f"wootax/{x+1}"] = Map({
"country":
Enum(["LL"]) if is_template else Enum(COUNTRIES),
"state":
Str(),
"rate":
Decimal(),
"name":
Str(),
"priority":
Int(),
"compound":
Int(),
"shipping":
Int(),
"order":
Int(),
"class":
Enum([""]) if x == 0 else Enum(tax_classes),
"locations":
Map({}),
})
try:
return yaml_load(yaml, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
@staticmethod
def load_string(data: bytes, schema, path: str):
"""Load and validate yaml data."""
try:
return yaml_load(data, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
_report_schema = Map({
Opt("sentry"):
Map({
Opt("dsn"):
Map({
Opt("value"): EmptyNone() | Str(),
Opt("fromFile"): EmptyNone() | Str(),
Opt("fromEnvVar"): EmptyNone() | Str(),
}),
Opt("fingerprint"):
Seq(Str()),
Opt("level"):
Str(),
Opt("extra"):
MapPattern(Str(),
Str() | Int() | Bool()),
Opt("body"):
Str(),
}),
Opt("mail"):
Map({
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
}),
})
_job_defaults_common = {
from inspect import signature, Parameter
from strictyaml import Map, Str, Int, Seq, Any, Optional, Float, EmptyNone, Bool
from strictyaml import dirty_load
# TODO: use stackoverflow.com/questions/37929851/parse-numpydoc-docstring-and-access-components
# for now just hardcode most common / important types
ARTM_TYPES = {
"tau": Float(),
"topic_names": Str() | Seq(Str()) | EmptyNone(),
# TODO: handle class_ids in model and in regularizers separately
"class_ids": Str() | Seq(Str()) | EmptyNone(),
"gamma": Float() | EmptyNone(),
"seed": Int(),
"num_document_passes": Int(),
"num_processors": Int(),
"cache_theta": Bool(),
"reuse_theta": Bool(),
"theta_name": Str()
}
element = Any()
base_schema = Map({
'regularizers': Seq(element),
'stages': Seq(element),
'model': Map({
"dataset_path": Str(),
"modalities_to_use": Seq(Str()),
"main_modality": Str()
}),
'topics': Map({
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
python_version=GivenProperty(Str()),
selenium_version=GivenProperty(Str()),
website=GivenProperty(MapPattern(Str(), Str())),
selectors_yml=GivenProperty(Str()),
javascript=GivenProperty(Str()),
setup=GivenProperty(Str()),
code=GivenProperty(Str()),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
)
def __init__(self, keypath, settings):
self.path = keypath
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
self.path.profile = self.path.gen.joinpath("profile")
dirtemplate.DirTemplate(
"webapp", self.path.key / "htmltemplate", self.path.state
).with_vars(javascript=self.given.get("javascript", "")).with_files(
base_html={
filename: {
"content": content
}
for filename, content in self.given.get("website", {}).items()
}).ensure_built()
self.path.state.joinpath("selectors.yml").write_text(
self.given["selectors.yml"])
self.server = (python("-m", "http.server").in_dir(self.path.state /
"webapp").pexpect())
self.server.expect("Serving HTTP on 0.0.0.0")
if not self.path.profile.exists():
self.path.profile.mkdir()
self.python = project_build(self.path, self.given["python version"],
self.given["selenium version"]).bin.python
self.example_py_code = (ExamplePythonCode(
self.python,
self.path.state).with_setup_code(self.given.get(
"setup", "")).with_terminal_size(160, 100).with_long_strings())
@validate(
code=Str(),
will_output=Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
raises=Map({
Optional("type"):
Map({
"in python 2": Str(),
"in python 3": Str()
})
| Str(),
Optional("message"):
Map({
"in python 2": Str(),
"in python 3": Str()
})
| Str(),
}),
in_interpreter=Bool(),
)
def run(self, code, will_output=None, raises=None, in_interpreter=False):
if in_interpreter:
code = "{0}\nprint(repr({1}))".format(
"\n".join(code.strip().split("\n")[:-1]),
code.strip().split("\n")[-1])
to_run = self.example_py_code.with_code(code)
if self.settings.get("cprofile"):
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug)))
result = (to_run.expect_exceptions().run()
if raises is not None else to_run.run())
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")])
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self.settings.get("rewrite"):
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2") else
exception_type["in python 3"])
if message is not None:
if not isinstance(message, str):
differential = True
message = (message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"])
try:
result.exception_was_raised(exception_type, message)
except ExpectedExceptionMessageWasDifferent:
if self.settings.get("rewrite") and not differential:
new_raises = raises.copy()
new_raises["message"] = result.exception.message
self.current_step.update(raises=new_raises)
else:
raise
def do_nothing(self):
pass
def pause(self, message="Pause"):
import IPython
IPython.embed()
def tear_down(self):
self.server.kill(signal.SIGTERM)
self.server.wait()
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
})
})
_job_defaults_common = {
Opt("shell"):
Str(),
Opt("concurrencyPolicy"):
Enum(['Allow', 'Forbid', 'Replace']),
Opt("captureStderr"):
Bool(),
Opt("captureStdout"):
Bool(),
Opt("saveLimit"):
Int(),
Opt("failsWhen"):
Map({
"producesStdout": Bool(),
Opt("producesStderr"): Bool(),
Opt("nonzeroReturn"): Bool(),
}),
Opt("onFailure"):
Map({
Opt("retry"):
Map({
"maximumRetries": Int(),
Str(),
ModelMetadataKeys.TARGET_TYPE:
Str(),
Optional(ModelMetadataKeys.ENVIRONMENT_ID):
Str(),
Optional(ModelMetadataKeys.VALIDATION):
Map({
"input": Str(),
Optional("targetName"): Str()
}),
Optional(ModelMetadataKeys.MODEL_ID):
Str(),
Optional(ModelMetadataKeys.DESCRIPTION):
Str(),
Optional(ModelMetadataKeys.MAJOR_VERSION):
Bool(),
Optional(ModelMetadataKeys.INFERENCE_MODEL):
Map({
Optional("targetName"): Str(),
Optional("positiveClassLabel"): Str(),
Optional("negativeClassLabel"): Str(),
Optional("classLabels"): Seq(Str()),
Optional("classLabelsFile"): Str(),
Optional("predictionThreshold"): Int(),
}),
Optional(ModelMetadataKeys.TRAINING_MODEL):
Map({Optional("trainOnProject"): Str()}),
Optional(ModelMetadataKeys.HYPERPARAMETERS):
Any(),
Optional(ModelMetadataKeys.VALIDATION_SCHEMA):
get_type_schema_yaml_validator(),
#: `repo` metadata keys.
REPO_KEYS = {
"name": Str(), # Downloader doesn't use this, but cogs.red might.
"short": Str(),
"description": Str(),
"install_msg": Str(),
"author": Seq(Str()),
}
#: Metadata keys common to `shared_fields` and `cogs` schemas.
COMMON_KEYS = {
Optional("min_bot_version"): RedVersion(),
Optional("max_bot_version"): RedVersion(),
Optional("min_python_version"): PythonVersion(),
Optional("hidden", False): Bool(),
Optional("disabled", False): Bool(),
Optional("type", "COG"): Enum(["COG", "SHARED_LIBRARY"]),
}
#: `shared_fields` metadata keys.
SHARED_FIELDS_KEYS = {
"install_msg": Str(),
"author": Seq(Str()),
**COMMON_KEYS,
}
#: `cogs` metadata keys.
COG_KEYS = {
"name": Str(), # Downloader doesn't use this but I can set friendlier name
"short": Str(),
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
yaml_snippet=GivenProperty(
Str(), document="yaml_snippet:\n```yaml\n{{ yaml_snippet }}\n```"),
yaml_snippet_1=GivenProperty(
Str(),
document="yaml_snippet_1:\n```yaml\n{{ yaml_snippet_1 }}\n```"),
yaml_snippet_2=GivenProperty(
Str(),
document="yaml_snippet_2:\n```yaml\n{{ yaml_snippet_2 }}\n```"),
modified_yaml_snippet=GivenProperty(
Str(),
document=
"modified_yaml_snippet:\n```yaml\n{{ modified_yaml_snippet }}\n```"
),
python_version=GivenProperty(Str()),
ruamel_version=GivenProperty(Str()),
setup=GivenProperty(Str(), document="```python\n{{ setup }}\n```"),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
fails_on_python_2=InfoProperty(schema=Bool()),
description=InfoProperty(schema=Str()),
experimental=InfoProperty(schema=Bool()),
)
def __init__(self,
keypath,
python_path=None,
rewrite=False,
cprofile=False):
self.path = keypath
self._python_path = python_path
self._rewrite = rewrite
self._cprofile = cprofile
def set_up(self):
"""Set up your applications and the test environment."""
self.path.profile = self.path.gen.joinpath("profile")
if not self.path.profile.exists():
self.path.profile.mkdir()
if not self._python_path:
self.python = hitchpylibrarytoolkit.project_build(
"strictyaml",
self.path,
self.given["python version"],
{
"ruamel.yaml": self.given["ruamel version"]
},
).bin.python
else:
self.python = Path(self._python_path)
assert self.python.exists()
self.example_py_code = (ExamplePythonCode(
self.python, self.path.gen).with_code(self.given.get(
"code", "")).with_setup_code(self.given.get(
"setup", "")).with_terminal_size(160, 100).with_strings(
yaml_snippet_1=self.given.get("yaml_snippet_1"),
yaml_snippet=self.given.get("yaml_snippet"),
yaml_snippet_2=self.given.get("yaml_snippet_2"),
modified_yaml_snippet=self.given.get(
"modified_yaml_snippet"),
))
@no_stacktrace_for(AssertionError)
@no_stacktrace_for(HitchRunPyException)
@validate(
code=Str(),
will_output=Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
raises=Map({
Optional("type"): CODE_TYPE,
Optional("message"): CODE_TYPE
}),
in_interpreter=Bool(),
)
def run(
self,
code,
will_output=None,
yaml_output=True,
raises=None,
in_interpreter=False,
):
if in_interpreter:
code = "{0}\nprint(repr({1}))".format(
"\n".join(code.strip().split("\n")[:-1]),
code.strip().split("\n")[-1])
to_run = self.example_py_code.with_code(code)
if self._cprofile:
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug)))
if raises is None:
result = (to_run.expect_exceptions().run()
if raises is not None else to_run.run())
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")])
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(
**{"will output": actual_output})
else:
raise
elif raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2") else
exception_type["in python 3"])
if message is not None:
if not isinstance(message, str):
differential = True
message = (message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"])
try:
result = to_run.expect_exceptions().run()
result.exception_was_raised(exception_type, message)
except ExpectedExceptionMessageWasDifferent as error:
if self._rewrite and not differential:
new_raises = raises.copy()
new_raises["message"] = result.exception.message
self.current_step.update(raises=new_raises)
else:
raise
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_success(self):
if self._rewrite:
self.new_story.save()
if self._cprofile:
self.python(
self.path.key.joinpath("printstats.py"),
self.path.profile.joinpath("{0}.dat".format(self.story.slug)),
).run()
# it is a place holder if user wants to add some fields and read them on his own
CUSTOM_PREDICTOR = "customPredictor"
MODEL_CONFIG_SCHEMA = Map(
{
ModelMetadataKeys.NAME: Str(),
ModelMetadataKeys.TYPE: Str(),
ModelMetadataKeys.TARGET_TYPE: Str(),
Optional(ModelMetadataKeys.ENVIRONMENT_ID): Str(),
Optional(ModelMetadataKeys.VALIDATION): Map(
{"input": Str(), Optional("targetName"): Str()}
),
Optional(ModelMetadataKeys.MODEL_ID): Str(),
Optional(ModelMetadataKeys.DESCRIPTION): Str(),
Optional(ModelMetadataKeys.MAJOR_VERSION): Bool(),
Optional(ModelMetadataKeys.INFERENCE_MODEL): Map(
{
"targetName": Str(),
Optional("positiveClassLabel"): Str(),
Optional("negativeClassLabel"): Str(),
Optional("classLabels"): Seq(Str()),
Optional("classLabelsFile"): Str(),
Optional("predictionThreshold"): Int(),
}
),
Optional(ModelMetadataKeys.TRAINING_MODEL): Map({Optional("trainOnProject"): Str()}),
Optional(ModelMetadataKeys.HYPERPARAMETERS): Any(),
Optional(ModelMetadataKeys.CUSTOM_PREDICTOR): Any(),
}
)
def build(self):
if self._build_path.exists():
self._build_path.rmtree()
self._build_path.mkdir()
if self._src_path.joinpath("dirtemplate.yml").exists():
config = load(
self._src_path.joinpath("dirtemplate.yml").text(),
Map({
Optional("base templates"):
Str(),
"templated":
Seq(
MapPattern(
Str(),
Map({
Optional("content"): Bool(),
Optional("filename"): Bool(),
}))),
})).data
else:
config = {"templated": []}
src_paths = list(pathquery(self._src_path))
templated_filenames = [
list(template.keys())[0] for template in config['templated']
]
if "base templates" in config:
templated_filenames.extend(
pathquery(self._src_path.joinpath(config['base templates'])))
non_templated = []
for srcpath in src_paths:
relpath = srcpath.relpath(self._src_path)
add = True
if relpath in templated_filenames:
add = False
if relpath == "dirtemplate.yml":
add = False
if srcpath.isdir():
add = False
if "base templates" in config:
if relpath.startswith(config['base templates']):
add = False
if add:
non_templated.append(relpath)
for relpath in non_templated:
dest_path = self._dest.joinpath(relpath)
if not dest_path.dirname().exists():
dest_path.dirname().makedirs()
if str(relpath) not in self._ignore_files:
self._src_path.joinpath(relpath).copy(dest_path)
for template_configuration in config['templated']:
for src_path in src_paths:
if not src_path.isdir():
relpath = src_path.relpath(self._src_path)
if relpath in template_configuration.keys():
if 'filename' in template_configuration[relpath]:
slug = slugify(relpath, separator=u'_')
if slug in self._files.keys():
for filename, variables in self._files[
slug].items():
dest_path = self._dest.joinpath(filename)
if not dest_path.dirname().exists():
dest_path.dirname().makedirs()
render_vars = {}
for name, var in self._render_vars.items():
render_vars[name] = var
for name, filevar in variables.items():
render_vars[name] = filevar
render_vars['thisdir'] = pathquery(
dest_path.dirname())
dest_path.write_text(
render(
src_path, self._functions,
render_vars,
base_templates(
self._src_path,
config.get("base templates"))))
else:
raise Exception(
("{0} templated filename exists but not "
"specified with with_files".format(
relpath)))
else:
dest_path = self._dest.joinpath(relpath)
if not dest_path.dirname().exists():
dest_path.dirname().makedirs()
render_vars = copy(self._render_vars)
render_vars['thisdir'] = pathquery(
dest_path.dirname())
if template_configuration[relpath]['content']:
dest_path.write_text(
render(
src_path,
self._functions,
render_vars,
base_templates(
self._src_path,
config.get("base templates")),
))
self.refingerprint()
parsed = load(s, schema_ver, label=fname)
except InconsistentIndentationDisallowed as e:
print('Use the same indentation across the file')
print(e)
sys.exit(1)
except YAMLError as e:
print('YAML parsing error:')
print(e)
sys.exit(1)
schema = Map({
"kiplot":
Map({"version": Int()}), # noqa: E127
Optional("preflight"):
Map({
Optional("run_drc"): Bool(), # noqa: E121
Optional("run_erc"): Bool(),
Optional("update_xml"): Bool(),
Optional("check_zone_fills"): Bool(),
Optional("ignore_unconnected"): Bool(),
}),
Optional("outputs"):
Seq(Any())
})
try:
parsed = load(s, schema, label=fname)
except YAMLError as e:
print('YAML parsing error:')
print(e)
sys.exit(1)
count_vocab_size,
handle_regularizer,
)
import artm
from strictyaml import Map, Str, Int, Seq, Float, Bool
from strictyaml import Any, Optional, EmptyDict, EmptyNone, EmptyList
from strictyaml import dirty_load
SUPPORTED_CUBES = [CubeCreator, RegularizersModifierCube]
SUPPORTED_STRATEGIES = [PerplexityStrategy, GreedyStrategy]
TYPE_VALIDATORS = {
'int': Int(),
'bool': Bool(),
'str': Str(),
'float': Float()
}
def choose_key(param):
"""
Parameters
----------
param : inspect.Parameter
Returns
-------
str or strictyaml.Optional
"""
# -*- coding:utf-8 -*-
# author: Xinge
from pathlib import Path
from strictyaml import Bool, Float, Int, Map, Seq, Str, as_document, load
model_params = Map({
"model_architecture": Str(),
"output_shape": Seq(Int()),
"fea_dim": Int(),
"out_fea_dim": Int(),
"num_class": Int(),
"num_input_features": Int(),
"use_norm": Bool(),
"init_size": Int(),
})
dataset_params = Map({
"dataset_type": Str(),
"pc_dataset_type": Str(),
"ignore_label": Int(),
"return_test": Bool(),
"fixed_volume_space": Bool(),
"label_mapping": Str(),
"max_volume_space": Seq(Float()),
"min_volume_space": Seq(Float()),
})
train_data_loader = Map({
"data_path": Str(),
from datarobot_drum.drum.exceptions import DrumCommonException
CONFIG_FILENAME = "model-metadata.yaml"
DR_LINK_FORMAT = "{}/model-registry/custom-models/{}"
MODEL_LOGS_LINK_FORMAT = "{url}/projects/{project_id}/models/{model_id}/log"
schema = Map(
{
"name": Str(),
"type": Str(),
"environmentID": Str(),
"targetType": Str(),
"validation": Map({"input": Str(), Optional("targetName"): Str()}),
Optional("modelID"): Str(),
Optional("description"): Str(),
Optional("majorVersion"): Bool(),
Optional("inferenceModel"): Map(
{
"targetName": Str(),
Optional("positiveClassLabel"): Str(),
Optional("negativeClassLabel"): Str(),
Optional("predictionThreshold"): Int(),
}
),
Optional("trainingModel"): Map({Optional("trainOnProject"): Str()}),
}
)
def _read_metadata(code_dir):
code_dir = Path(code_dir)
class Experiment:
"""
This class orchestrates the analysis pipeline for our redox imaging experiments.
"""
experiment_schema = Map({
"pipeline":
Map({
"strategy": Str(),
"acquisition_method": Enum(["acquire", "mda"]),
"trimmed_profile_length": Int(),
"untrimmed_profile_length": Int(),
"seg_threshold": Int(),
"measurement_order": Int(),
"measure_thickness": Float(),
"reference_wavelength": Str(),
"image_register": Int(),
"channel_register": Int(),
"population_register": Int(),
"trimmed_regions": MapPattern(Str(), CommaSeparated(Float())),
"untrimmed_regions": MapPattern(Str(), CommaSeparated(Float())),
}),
"redox":
Map({
"ratio_numerator": Str(),
"ratio_denominator": Str(),
"r_min": Float(),
"r_max": Float(),
"instrument_factor": Float(),
"midpoint_potential": Float(),
"z": Int(),
"temperature": Float(),
}),
"registration":
Map({
"n_deriv": Float(),
"warp_n_basis": Float(),
"warp_order": Float(),
"warp_lambda": Float(),
"smooth_lambda": Float(),
"smooth_n_breaks": Float(),
"smooth_order": Float(),
"rough_lambda": Float(),
"rough_n_breaks": Float(),
"rough_order": Float(),
}),
"output":
Map({
"should_save_plots": Bool(),
"should_save_profile_data": Bool(),
"should_save_summary_data": Bool(),
}),
})
seg_images: xr.DataArray = None
rot_fl: xr.DataArray = None
rot_seg: xr.DataArray = None
midlines: xr.DataArray = None
untrimmed_raw_profiles: xr.DataArray = None
untrimmed_std_profiles: xr.DataArray = None
untrimmed_reg_profiles: xr.DataArray = None
trimmed_raw_profiles: xr.DataArray = None
trimmed_std_profiles: xr.DataArray = None
trimmed_reg_profiles: xr.DataArray = None
channel_warps: xr.DataArray = None
std_warps: xr.DataArray = None
def __init__(self, exp_dir):
self.experiment_dir = Path(exp_dir)
self.settings_path = self.experiment_dir.joinpath("settings.yaml")
try:
with open(self.settings_path, "r") as f:
self.config = load(f.read(), self.experiment_schema).data
except YAMLError:
raise ValueError("Incorrectly specified config file.")
self.experiment_id = self.experiment_dir.stem
# compute the filenames/paths for this experiment
self.movement_path = self.experiment_dir.joinpath(self.experiment_id +
"-mvmt.csv")
self.frame_map_path = self.experiment_dir.joinpath(self.experiment_id +
"-frame_map.csv")
self.processed_images_dir = self.experiment_dir.joinpath(
"processed_images")
self.rot_seg_dir = self.processed_images_dir.joinpath("rot_seg")
self.rot_fl_dir = self.processed_images_dir.joinpath("rot_fl")
self.fl_imgs_dir = self.processed_images_dir.joinpath(
"fluorescent_images")
self.orig_images_path = self.processed_images_dir.joinpath("images.nc")
self.seg_images_path = self.processed_images_dir.joinpath(
"seg_images.nc")
self.aligned_images_path = self.processed_images_dir.joinpath(
"aligned_images.nc")
self.aligned_seg_images_path = self.processed_images_dir.joinpath(
"aligned_seg_images.nc")
# load images
self.images = self._load_raw_images()
# try to load masks
try:
self.load_masks()
except IOError:
logging.info("No masks found in experiment directory")
pass
# Computed Filepaths
@property
def midlines_path(self) -> Path:
return self.analysis_dir.joinpath("midlines.pickle")
@property
def raw_img_stack_path(self) -> Path:
# TODO test that this works
accepted_extensions = [".tif", ".tiff", ".stk"]
candidate_paths = [
self.experiment_dir.joinpath(f"{self.experiment_id}{ext}")
for ext in accepted_extensions
]
for path in candidate_paths:
if path.exists():
return path
raise ValueError(
f"No image found in experiment directory. Tried the following files: {candidate_paths}"
)
@property
def fig_dir(self):
return self.analysis_dir.joinpath("figs")
def untrimmed_profile_data_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-untrimmed_{treatment}_profile_data.nc")
def trimmed_profile_data_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-trimmed_{treatment}_profile_data.nc")
@property
def channel_warp_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-channel_warps.nc")
@property
def std_warp_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id + "-std_warps.nc")
def untrimmed_profile_data_csv_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-untrimmed_{treatment}_profile_data.csv")
def trimmed_profile_data_csv_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-trimmed_{treatment}_profile_data.csv")
@property
def untrimmed_region_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-untrimmed_region_data.csv")
@property
def trimmed_region_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-trimmed_region_data.csv")
@property
def analysis_dir(self) -> Path:
date_str = datetime.datetime.now().strftime("%Y-%m-%d")
strategy = self.config["pipeline"]["strategy"]
if len(strategy) > 0:
suffix = f"_{strategy}"
else:
suffix = ""
analysis_dir_ = self.experiment_dir.joinpath(
"analyses",
utils.get_valid_filename(f"{date_str}{suffix}"),
)
# analysis_dir_.mkdir(parents=True, exist_ok=True)
return analysis_dir_
def _load_raw_images(self):
"""
This returns the raw (non-median-subtracted) images
"""
logging.info(f"Loading image data from {self.raw_img_stack_path}")
raw_image_data = pio.load_tiff_as_hyperstack(
img_stack_path=self.raw_img_stack_path,
manual_metadata=self.frame_map_path,
mvmt_metadata=self.movement_path,
)
raw_image_data = raw_image_data.assign_coords({
"experiment_id": (
("animal", ),
np.repeat(self.experiment_id, raw_image_data.animal.size),
)
})
raw_image_data = self.add_experiment_metadata_to_data_array(
raw_image_data)
return raw_image_data
def _load_movement(self) -> pd.DataFrame:
movement_path = self.experiment_dir.joinpath(self.experiment_id +
"-mvmt.csv")
try:
df = pd.read_csv(movement_path)
df = df.pivot_table(index="animal",
columns=["region", "pair"],
values="movement")
df = df.stack("pair")
return df
except FileNotFoundError:
logging.warning(
f"Tried to access {movement_path}; file was not found")
return None
def make_analysis_dir(self) -> None:
logging.info(f"Making analysis directory at {self.analysis_dir}")
self.analysis_dir.mkdir(parents=True, exist_ok=True)
@property
def trimmed_summary_table(self):
df = profile_processing.summarize_over_regions(
self.trimmed_raw_profiles,
regions=self.config["pipeline"]["trimmed_regions"],
rescale=False,
**self.config["redox"],
)
return df
@property
def untrimmed_summary_table(self):
df = profile_processing.summarize_over_regions(
self.untrimmed_raw_profiles,
regions=self.config["pipeline"]["untrimmed_regions"],
**self.config["redox"],
)
return df
####################################################################################
# PIPELINE
####################################################################################
def full_pipeline(self):
logging.info(f"Starting full pipeline run for {self.experiment_dir}")
self.make_analysis_dir()
logging.info(f"Saving fluorescent images to {self.fl_imgs_dir}")
pio.save_images_xarray_to_tiffs(self.images,
self.fl_imgs_dir,
prefix=self.experiment_id)
self.segment_pharynxes()
self.register_images()
self.align_and_center()
self.calculate_midlines()
self.measure_under_midlines()
self.register_profiles()
self.trim_data()
self.calculate_redox()
self.do_manual_ap_flips()
self.persist_to_disk()
logging.info(f"Finished full pipeline run for {self.experiment_dir}")
return self
def run_neuron_pipeline(self):
logging.info(
f"Starting full neuron analysis pipeline run for {self.experiment_dir}"
)
self.make_analysis_dir()
df = ip.measure_under_labels(self.images,
self.seg_images).reset_index()
df.to_csv(self.analysis_dir /
(self.experiment_id + "-neuron_analysis.csv"))
def segment_pharynxes(self):
if self.seg_images is not None:
logging.info("masks have been specified. skipping mask generation")
self.save_masks()
return
else:
logging.info("Generating masks")
self.seg_images = ip.segment_pharynxes(
self.images,
wvl=self.config["pipeline"]["reference_wavelength"])
self.save_masks()
def register_images(self):
if self.config["pipeline"]["image_register"]:
logging.info("Registering Images")
self.images = ip.register_all_images(self.images, self.seg_images)
def align_and_center(self):
logging.info("Centering and rotating pharynxes")
self.rot_fl, self.rot_seg = ip.center_and_rotate_pharynxes(
self.images,
self.seg_images,
)
logging.info(f"Saving rotated FL images to {self.aligned_images_path}")
pio.save_profile_data(self.rot_fl, self.aligned_images_path)
logging.info(f"Saving rotated masks to {self.aligned_seg_images_path}")
pio.save_profile_data(self.rot_seg, self.aligned_seg_images_path)
def calculate_midlines(self):
logging.info("Calculating midlines")
self.midlines = ip.calculate_midlines(self.rot_seg, degree=4)
def measure_under_midlines(self):
logging.info("Measuring under midlines")
self.untrimmed_raw_profiles = ip.measure_under_midlines(
self.rot_fl,
self.midlines,
n_points=self.config["pipeline"]["untrimmed_profile_length"],
order=self.config["pipeline"]["measurement_order"],
thickness=float(self.config["pipeline"]["measure_thickness"]),
)
self.untrimmed_raw_profiles = profile_processing.align_pa(
self.untrimmed_raw_profiles)
self.untrimmed_raw_profiles = self.add_experiment_metadata_to_data_array(
self.untrimmed_raw_profiles)
# subtract the image medians from the profile data
logging.info("Subtracting image medians from profile data")
self.untrimmed_raw_profiles = ip.subtract_medians(
self.untrimmed_raw_profiles, self.images)
def register_profiles(self):
if self.config["pipeline"]["population_register"]:
logging.info("Standardizing profiles")
(
self.untrimmed_std_profiles,
self.std_warps,
) = profile_processing.standardize_profiles(
self.untrimmed_raw_profiles,
redox_params=self.config["redox"],
**self.config["registration"],
)
if self.config["pipeline"]["channel_register"]:
logging.info("Channel-Registering profiles")
if self.untrimmed_std_profiles is not None:
logging.info(
"using the standardize profiles for channel-registration")
data_to_register = self.untrimmed_std_profiles
else:
logging.info("using the raw profiles for channel-registration")
data_to_register = self.untrimmed_raw_profiles
(
self.untrimmed_reg_profiles,
self.channel_warps,
) = profile_processing.channel_register(
data_to_register,
redox_params=self.config["redox"],
reg_params=self.config["registration"],
)
def trim_data(self):
logging.info("Trimming intensity data")
self.trimmed_raw_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_raw_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
if self.untrimmed_std_profiles is not None:
self.trimmed_std_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_std_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
if self.untrimmed_reg_profiles is not None:
self.trimmed_reg_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_reg_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
def calculate_redox(self):
logging.info("Calculating redox measurements")
redox_params = self.config["redox"]
# Images
self.images = utils.add_derived_wavelengths(self.images,
**redox_params)
self.rot_fl = utils.add_derived_wavelengths(self.rot_fl,
**redox_params)
# profiles
self.trimmed_raw_profiles = utils.add_derived_wavelengths(
self.trimmed_raw_profiles, **redox_params)
self.untrimmed_raw_profiles = utils.add_derived_wavelengths(
self.untrimmed_raw_profiles, **redox_params)
def do_manual_ap_flips(self):
# TODO finish implementation
logging.info("skipping manual AP flips - not implemented")
def flip_at(self, idx):
# TODO finish implementation
raise NotImplementedError
####################################################################################
# PERSISTENCE / IO
####################################################################################
def save_images(self):
"""Save this experiment's images to disk as netCDF4 files"""
imgs_paths = [
(self.images, self.orig_images_path),
(self.rot_fl, self.aligned_images_path),
(self.seg_images, self.seg_images_path),
(self.rot_seg, self.aligned_seg_images_path),
]
for img, path in imgs_paths:
if img is not None:
logging.info(f"Saving images to {path}")
img.to_netcdf(path)
# def load_tiff_as_hyperstack(self):
# pass
def make_fig_dir(self):
fig_dir = self.analysis_dir.joinpath("figs")
fig_dir.mkdir(parents=True, exist_ok=True)
return fig_dir
def save_individual_profiles(self, profile_data, treatment: str,
trimmed: bool):
if profile_data is None:
return
fig_dir = self.make_fig_dir()
profile_data_fig_dir = (fig_dir / "profile_data" / treatment /
("trimmed" if trimmed else "untrimmed"))
individual_data_fig_dir = profile_data_fig_dir.joinpath("inividual")
individual_data_fig_dir.mkdir(exist_ok=True, parents=True)
for title, fig in plots.generate_wvl_pair_timepoint_profile_plots(
profile_data):
title = title.replace(" ", "")
fig.savefig(individual_data_fig_dir /
f"{self.experiment_id}-{title}-individuals.pdf")
plt.close(fig)
def save_avg_profiles(self, profile_data, treatment: str, trimmed: bool):
if profile_data is None:
return
fig_dir = self.make_fig_dir()
profile_data_fig_dir = (fig_dir / "profile_data" / treatment /
("trimmed" if trimmed else "untrimmed"))
individual_data_fig_dir = profile_data_fig_dir.joinpath("avg")
individual_data_fig_dir.mkdir(exist_ok=True, parents=True)
for title, fig in plots.generate_avg_wvl_pair_profile_plots(
profile_data):
title = title.replace(" ", "")
fig.savefig(individual_data_fig_dir /
f"{self.experiment_id}-{title}-avg.pdf")
plt.close(fig)
def save_plots(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
for data, treatment, trimmed in [
(self.untrimmed_raw_profiles, "raw", False),
(self.untrimmed_std_profiles, "standardized", False),
(self.untrimmed_reg_profiles, "channel-registered", False),
(self.trimmed_raw_profiles, "raw", True),
(self.trimmed_std_profiles, "standardized", True),
(self.trimmed_reg_profiles, "channel-registered", True),
]:
self.save_individual_profiles(data, treatment, trimmed)
self.save_avg_profiles(data, treatment, trimmed)
# frame-normed Ratio Images
mvmt_annotation_img_path = self.fig_dir.joinpath(
f"{self.experiment_id}-movement_annotation_imgs.pdf")
imgs = utils.add_derived_wavelengths(self.images,
**self.config["redox"])
with PdfPages(mvmt_annotation_img_path) as pdf:
for i in tqdm(range(self.images.animal.size)):
fig = plots.plot_pharynx_R_imgs(imgs[i],
mask=self.seg_images[i])
fig.suptitle(f"animal = {i}")
pdf.savefig(fig)
if (i % 20) == 0:
plt.close("all")
# Pop-normed ratio images
u = self.trimmed_raw_profiles.sel(wavelength="r").mean()
std = self.trimmed_raw_profiles.sel(wavelength="r").std()
for pair in self.rot_fl.pair.values:
for tp in self.rot_fl.timepoint.values:
ratio_img_path = self.fig_dir.joinpath(
f"{self.experiment_id}-ratio_images-pair={pair};timepoint={tp}.pdf"
)
with PdfPages(ratio_img_path) as pdf:
logging.info(
f"Saving ratio images to {ratio_img_path}")
for i in tqdm(range(self.rot_fl.animal.size)):
fig, ax = plt.subplots(dpi=300)
ratio_img = (self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_numerator"],
pair=pair,
timepoint=tp,
) / self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_denominator"],
pair=pair,
timepoint=tp,
))[i]
fl_img = self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_numerator"],
pair=pair,
timepoint=tp,
)[i]
im, cbar = plots.imshow_ratio_normed(
ratio_img,
fl_img,
r_min=u - (std * 1.96),
r_max=u + (std * 1.96),
colorbar=True,
i_max=5000,
i_min=1000,
ax=ax,
)
ax.plot(
*self.midlines.sel(
pair=pair,
timepoint=tp,
)[i].values[()].linspace(),
color="green",
alpha=0.3,
)
strain = self.rot_fl.strain.values[i]
ax.set_title(
f"Animal={i} ; Pair={pair} ; Strain={strain}")
cax = cbar.ax
for j in range(len(self.trimmed_raw_profiles)):
cax.axhline(
self.trimmed_raw_profiles.sel(
wavelength="r",
pair=pair,
timepoint=tp)[j].mean(),
color="k",
alpha=0.1,
)
cax.axhline(
self.trimmed_raw_profiles.sel(
wavelength="r", pair=pair,
timepoint=tp)[i].mean(),
color="k",
)
pdf.savefig()
if (i % 20) == 0:
plt.close("all")
def persist_profile_data(self):
for treatment, untrimmed_profile_data in (
("raw", self.untrimmed_raw_profiles),
("std", self.untrimmed_std_profiles),
("reg", self.untrimmed_reg_profiles),
):
if untrimmed_profile_data is not None:
untrimmed_prof_path = self.untrimmed_profile_data_path(
treatment)
logging.info(
f"Saving untrimmed {treatment} profile data to {untrimmed_prof_path}"
)
pio.save_profile_data(untrimmed_profile_data,
untrimmed_prof_path)
untrimmed_prof_path_csv = self.untrimmed_profile_data_csv_path(
treatment)
profile_processing.to_dataframe(
untrimmed_profile_data,
"value").to_csv(untrimmed_prof_path_csv)
for treatment, trimmed_profile_data in (
("raw", self.trimmed_raw_profiles),
("std", self.trimmed_std_profiles),
("reg", self.trimmed_reg_profiles),
):
if trimmed_profile_data is not None:
trimmed_prof_path = self.trimmed_profile_data_path(treatment)
logging.info(
f"Saving trimmed {treatment} profile data to {trimmed_prof_path}"
)
pio.save_profile_data(trimmed_profile_data, trimmed_prof_path)
trimmed_prof_path_csv = self.trimmed_profile_data_csv_path(
treatment)
logging.info(
f"Saving trimmed {treatment} profile data to {trimmed_prof_path_csv}"
)
profile_processing.to_dataframe(
trimmed_profile_data,
"value").to_csv(trimmed_prof_path_csv)
# Warps, if necessary
if self.config["pipeline"]["channel_register"]:
logging.info(
f"Saving channel warp data to {self.channel_warp_data_path}")
self.channel_warps.to_netcdf(self.channel_warp_data_path)
if self.config["pipeline"]["population_register"]:
logging.info(
f"Saving channel warp data to {self.std_warp_data_path}")
self.std_warps.to_netcdf(self.std_warp_data_path)
def save_summary_data(self):
# Persist the region means
logging.info(
f"Saving untrimmed region means to {self.untrimmed_region_data_path}"
)
self.untrimmed_summary_table.to_csv(self.untrimmed_region_data_path)
logging.info(
f"Saving trimmed region means to {self.trimmed_region_data_path}")
self.trimmed_summary_table.to_csv(self.trimmed_region_data_path)
def save_masks(self):
logging.info(f"saving masks to {self.seg_images_path}")
pio.save_profile_data(self.seg_images, self.seg_images_path)
def load_masks(self):
self.seg_images = pio.load_profile_data(self.seg_images_path)
logging.info(f"Loaded masks from {self.seg_images_path}")
def save_midlines(self):
pio.save_midlines(self.midlines_path, self.midlines)
def load_midlines(self):
return pio.load_midlines(self.midlines_path)
def persist_to_disk(self):
logging.info(
f"Saving {self.experiment_id} inside {self.experiment_dir}")
self.save_midlines()
if self.config["output"]["should_save_summary_data"]:
self.save_summary_data()
if self.config["output"]["should_save_profile_data"]:
self.persist_profile_data()
if self.config["output"]["should_save_plots"]:
self.save_plots()
####################################################################################
# MISC / HELPER
####################################################################################
def add_experiment_metadata_to_data_array(self, data_array: xr.DataArray):
params = {}
params.update(self.config["pipeline"])
params.update(self.config["redox"])
params.update(self.config["registration"])
to_remove = ["trimmed_regions", "untrimmed_regions"]
for k in to_remove:
del params[k]
return data_array.assign_attrs(**params)
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
scripts=GivenProperty(MapPattern(Str(), Str())),
python_version=GivenProperty(Str()),
pexpect_version=GivenProperty(Str()),
icommandlib_version=GivenProperty(Str()),
setup=GivenProperty(Str()),
files=GivenProperty(MapPattern(Str(), Str())),
code=GivenProperty(Str()),
)
info_definition = InfoDefinition(
importance=InfoProperty(schema=Int()),
docs=InfoProperty(schema=Str()),
fails_on_python_2=InfoProperty(schema=Bool()),
)
def __init__(self, keypath, rewrite=False):
self.path = keypath
self._rewrite = rewrite
self._cprofile = False
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
for script in self.given.get("scripts", []):
script_path = self.path.state.joinpath(script)
if not script_path.dirname().exists():
script_path.dirname().makedirs()
script_path.write_text(self.given["scripts"][script])
script_path.chmod("u+x")
for filename, contents in self.given.get("files", {}).items():
self.path.state.joinpath(filename).write_text(contents)
self.python = hitchpylibrarytoolkit.project_build(
"commandlib", self.path, self.given["python version"]
).bin.python
self.example_py_code = (
ExamplePythonCode(self.python, self.path.state)
.with_code(self.given.get("code", ""))
.with_setup_code(self.given.get("setup", ""))
)
def _story_friendly_output(self, text):
return text.replace(self.path.state, "/path/to")
@no_stacktrace_for(AssertionError)
@no_stacktrace_for(HitchRunPyException)
@validate(
code=Str(),
will_output=Str(),
raises=Map(
{
Optional("type"): Map({"in python 2": Str(), "in python 3": Str()})
| Str(),
Optional("message"): Map({"in python 2": Str(), "in python 3": Str()})
| Str(),
}
),
)
def run(self, code, will_output=None, raises=None):
to_run = self.example_py_code.with_code(code)
if self._cprofile:
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug))
)
result = (
to_run.expect_exceptions().run() if raises is not None else to_run.run()
)
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")]
)
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2")
else exception_type["in python 3"]
)
if message is not None:
if not isinstance(message, str):
differential = True
message = (
message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"]
)
try:
result = self.example_py_code.expect_exceptions().run()
result.exception_was_raised(exception_type)
exception_message = self._story_friendly_output(
result.exception.message
)
Templex(exception_message).assert_match(message)
except AssertionError:
if self._rewrite and not differential:
new_raises = raises.copy()
new_raises["message"] = self._story_friendly_output(
result.exception.message
)
self.current_step.update(raises=new_raises)
else:
raise
def file_contents_will_be(self, filename, contents):
file_contents = "\n".join(
[
line.rstrip()
for line in self.path.state.joinpath(filename)
.bytes()
.decode("utf8")
.strip()
.split("\n")
]
)
try:
# Templex(file_contents).assert_match(contents.strip())
assert file_contents == contents.strip(), "{0} not {1}".format(
file_contents, contents.strip()
)
except AssertionError:
if self._rewrite:
self.current_step.update(contents=file_contents)
else:
raise
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_success(self):
if self._cprofile:
self.python(
self.path.key.joinpath("printstats.py"),
self.path.profile.joinpath("{0}.dat".format(self.story.slug)),
).run()
# конфиг которым управляют админы
CONFIG_ENV_NAME = 'games_config'
if CONFIG_ENV_NAME in os.environ:
src_file = os.environ.get(CONFIG_ENV_NAME)
else:
src_file = '/etc/games/games.yml'
with open(src_file, 'r') as file_h:
CFG_CONTENT = file_h.read()
SCHEMA = Map({
"connections":
Map({
"pgsql": Map({"uri": Str()}),
"celery": Map({"uri": Str()}),
}),
"static_root":
Str(),
"api_docs_enabled":
Bool(),
"debug":
Bool(),
"secret_key":
Str(),
"media_root":
Str()
})
config = load(CFG_CONTENT, SCHEMA)