def load_scheme(yaml_text):
'''
Some optional keys have enforced default values, otherwise use dict.get()
'''
schema = MapPattern(
Str(),
Map({
'description':
Str(),
Optional('alias'):
Str(),
'version':
Str(),
'directives':
MapPattern(
Str(),
Map({
Optional('description'):
Str(),
'formats':
Seq(Enum(FORMATS)),
Optional('prepare'):
Map({
Optional('remove_whitespace', default=False): Bool(),
Optional('remove_characters', default=['']):
Seq(Str()),
Optional('strip_characters', default=['']): Seq(Str()),
}),
Optional('validate'):
Map({
Optional('alphabet'): Enum(ALPHABETS),
Optional('min_length'): Int(),
Optional('max_length'): Int(),
}),
Optional('target'):
Str(),
Optional('helper', default=False):
Bool()
}),
),
'algorithm':
Enum(ALGORITHMS),
'encodings':
MapPattern(
Str(),
Map({
'type':
Enum(ENCODINGS),
Optional('length', default=0):
Int(),
Optional('prefix', default=''):
Str(),
Optional('separator'):
Map({
'character': Str(),
'interval': Int()
})
}))
}))
return load(yaml_text, schema)
def __init__(self, filename):
"""Load config from YAML file."""
filename = path.abspath(filename)
if filename is None:
self._config = []
else:
try:
with open(filename, 'r') as handle:
self._yaml = handle.read()
self._config = load(
self._yaml,
Seq(
Map({
Optional("name"):
"name",
"request":
Map({
Optional("path"):
Str(),
Optional("method"):
Enum([
"get",
"post",
"put",
"delete",
"GET",
"POST",
"PUT",
"DELETE",
]),
Optional("headers"):
MapPattern(Str(), Str()),
Optional("data"):
Str(),
}),
"response":
Map({
"content": Str() | Map({"file": Str()}),
Optional("code"): Int(),
Optional("headers"): MapPattern(Str(), Str()),
}),
})))
except Exception as e:
sys.stderr.write(
"Error reading YAML config file: {0}\n".format(str(e)))
sys.exit(1)
# Read and store all references to external content files
for pair in self._config:
content = pair.get('response', {}).get('content')
if type(content) != str and "file" in content:
with open(
path.join(path.dirname(filename), content['file']),
'r') as content_file_handle:
pair['response']['content'] = \
content_file_handle.read()
def _get_schema(cls):
return Map({
PHASE_DEFINITIONS_TAG:
MapPattern(Str(), Map(cls._get_phase_mapping())),
ROUTE_DEFINITIONS_TAG:
MapPattern(Str(), Map(cls._get_route_mapping())),
MISSION_DEFINITION_TAG:
Map(cls._get_mission_mapping()),
})
def _get_schema(cls):
"""Schema of the whole mission file."""
return Map({
PHASE_DEFINITIONS_TAG:
MapPattern(Str(), cls._get_phase_schema()),
ROUTE_DEFINITIONS_TAG:
MapPattern(Str(), cls._get_route_schema()),
MISSION_DEFINITION_TAG:
MapPattern(Str(), cls._get_mission_schema()),
})
def load_data(filename):
schema = MapPattern(
Regex(u'[A-Za-z. ]+'),
MapPattern(
Regex(u'[A-Za-z\\-. ]+'),
Map({
'Description': Regex('.+'),
'Free': Bool(),
'Link': Url(),
}, Regex(u'.+'))))
with open(filename) as f:
return strictyaml.load(f.read(), schema).data
def __init__(self, sqlite_filename, templates):
self._db = Database(sqlite_filename)
self._templates = load(
Path(templates).bytes().decode('utf8'),
Map({
"documents": MapPattern(Str(), Str()),
"steps": MapPattern(Str(), Str()),
}),
).data
self._stories = [
Story(recording, self._db, self._templates)
for recording in self._db.Recording.select()
]
def parse_yaml(yaml_text):
selectors = load(
yaml_text,
MapPattern(Str(), Map({"appears when": Str(), "elements": ELEMENTS_SCHEMA})),
)
for page, page_properties in selectors.items():
revalidate_subelements(page_properties["elements"])
return selectors
def _get_info(cls):
# https://hitchdev.com/strictyaml
schema = Map({
Optional("knobs"):
Map({
Optional("log_to_console"): Bool(),
Optional("log_level_debug"): Bool(),
})
| EmptyDict(),
"globals":
Map({
"topic_prefix":
Str(),
Optional(
"reconnect_interval",
default=const.MQTT_DEFAULT_RECONNECT_INTERVAL,
):
Float(),
Optional("poll_interval",
default=const.MYQ_DEFAULT_POLL_INTERVAL):
Float(),
Optional(
"periodic_mqtt_report",
default=const.DEFAULT_PERIODIC_MQTT_REPORT,
):
Float(),
Optional("user_agent"):
Str(),
}),
"mqtt":
Map({
"host":
Str(),
Optional("client_id", default=const.MQTT_DEFAULT_CLIENT_ID):
Str(),
Optional("username"):
Str(),
Optional("password"):
Str(),
}),
"myq":
Map({
"email": Email(),
"password": Str(),
}),
Optional("alias"):
MapPattern(Str(), Str()) | EmptyDict(),
})
if not cls._info:
config_filename = cls._get_config_filename()
logger.info("loading yaml config file %s", config_filename)
with open(config_filename, "r") as ymlfile:
raw_cfg = load(ymlfile.read(), schema).data
cls._parse_raw_cfg(raw_cfg)
return cls._info
def test_operations_yaml_schema(self):
try:
load(
Path("../operations.yaml").bytes().decode('utf8'),
MapPattern(Str(), Map({
"command": Str(),
"tags": Str()
})))
except YAMLError as error:
self.fail("Something wrong with your schema")
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
scripts=GivenProperty(MapPattern(Str(), Str())),
python_version=GivenProperty(Str()),
pexpect_version=GivenProperty(Str()),
icommandlib_version=GivenProperty(Str()),
setup=GivenProperty(Str()),
files=GivenProperty(MapPattern(Str(), Str())),
code=GivenProperty(Str()),
)
info_definition = InfoDefinition(
importance=InfoProperty(schema=Int()),
docs=InfoProperty(schema=Str()),
fails_on_python_2=InfoProperty(schema=Bool()),
)
def __init__(self, keypath, rewrite=False):
self.path = keypath
self._rewrite = rewrite
self._cprofile = False
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
for script in self.given.get("scripts", []):
script_path = self.path.state.joinpath(script)
if not script_path.dirname().exists():
script_path.dirname().makedirs()
script_path.write_text(self.given["scripts"][script])
script_path.chmod("u+x")
for filename, contents in self.given.get("files", {}).items():
self.path.state.joinpath(filename).write_text(contents)
self.python = hitchpylibrarytoolkit.project_build(
"commandlib", self.path, self.given["python version"]
).bin.python
self.example_py_code = (
ExamplePythonCode(self.python, self.path.state)
.with_code(self.given.get("code", ""))
.with_setup_code(self.given.get("setup", ""))
)
def _story_friendly_output(self, text):
return text.replace(self.path.state, "/path/to")
@no_stacktrace_for(AssertionError)
@no_stacktrace_for(HitchRunPyException)
@validate(
code=Str(),
will_output=Str(),
raises=Map(
{
Optional("type"): Map({"in python 2": Str(), "in python 3": Str()})
| Str(),
Optional("message"): Map({"in python 2": Str(), "in python 3": Str()})
| Str(),
}
),
)
def run(self, code, will_output=None, raises=None):
to_run = self.example_py_code.with_code(code)
if self._cprofile:
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug))
)
result = (
to_run.expect_exceptions().run() if raises is not None else to_run.run()
)
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")]
)
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2")
else exception_type["in python 3"]
)
if message is not None:
if not isinstance(message, str):
differential = True
message = (
message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"]
)
try:
result = self.example_py_code.expect_exceptions().run()
result.exception_was_raised(exception_type)
exception_message = self._story_friendly_output(
result.exception.message
)
Templex(exception_message).assert_match(message)
except AssertionError:
if self._rewrite and not differential:
new_raises = raises.copy()
new_raises["message"] = self._story_friendly_output(
result.exception.message
)
self.current_step.update(raises=new_raises)
else:
raise
def file_contents_will_be(self, filename, contents):
file_contents = "\n".join(
[
line.rstrip()
for line in self.path.state.joinpath(filename)
.bytes()
.decode("utf8")
.strip()
.split("\n")
]
)
try:
# Templex(file_contents).assert_match(contents.strip())
assert file_contents == contents.strip(), "{0} not {1}".format(
file_contents, contents.strip()
)
except AssertionError:
if self._rewrite:
self.current_step.update(contents=file_contents)
else:
raise
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_success(self):
if self._cprofile:
self.python(
self.path.key.joinpath("printstats.py"),
self.path.profile.joinpath("{0}.dat".format(self.story.slug)),
).run()
class WooSchema:
"""Schema for localization YAML files."""
# https://github.com/woocart/woocart-defaults/blob/master/src/importers/class-woopage.php#L14
productMeta = {
"title": Str(),
"description": Str(),
Optional("price"): Str(),
Optional("category"): Str(),
"images": Seq(Str()),
}
# https://github.com/woocart/woocart-defaults/blob/master/src/importers/class-woopage.php#L14
pageMeta = {
"post_title": Str(),
Optional("post_name"): Str(),
Optional("post_excerpt"): Str(),
"post_status": Enum(["draft", "publish"]),
"post_type": Enum(["page", "post"]),
Optional("post_category"): Str(),
Optional("meta_input"): MapPattern(Str(), Str()),
Optional("woocart_defaults"): MapPattern(Str(), Str()),
}
localization = {
"woo/woocommerce_default_country":
Enum(COUNTRIES),
"wp/date_format":
Enum(["d/m/Y", "Y-m-d", "F j, Y", "m/d/Y"]),
"wp/time_format":
Enum(["H:i", "g:i A"]),
"wp/start_of_week":
Enum(["1", "2", "3", "4", "5", "6", "7"]),
"wp/timezone_string":
Enum(TIMEZONES),
"wp/blog_charset":
Enum(["UTF-8"]),
"wp/DEFAULT_WPLANG":
Enum(WPLANGS),
Optional("wp/blogdescription"):
Str(),
Optional("wp/woocommerce_demo_store_notice"):
Str(),
"woo/woocommerce_weight_unit":
Enum(["kg", "k", "lbs", "oz"]),
"woo/woocommerce_dimension_unit":
Enum(["m", "cm", "mm", "in", "yd"]),
"woo/woocommerce_currency":
Enum(CURRENCIES),
"woo/woocommerce_currency_pos":
Enum(["right_space", "left_space", "left", "right"]),
"woo/woocommerce_price_thousand_sep":
Enum([".", ","]),
"woo/woocommerce_price_decimal_sep":
Enum([",", "."]),
"woo/woocommerce_price_num_decimals":
Enum(["2"]),
Optional("woo/woocommerce_tax_classes"):
Seq(Str()),
"woo/woocommerce_bacs_settings":
Map({
"enabled": Bool(),
Optional("title"): Str(),
Optional("description"): Str(),
Optional("instructions"): Str(),
Optional("account_name"): Str(),
Optional("account_number"): Str(),
Optional("sort_code"): Str(),
Optional("bank_name"): Str(),
Optional("iban"): Str(),
Optional("bic"): Str(),
Optional("account_details"): Str(),
}),
"woo/woocommerce_cod_settings":
Map({
"enabled": Bool(),
Optional("title"): Str(),
Optional("description"): Str(),
Optional("instructions"): Str(),
Optional("enable_for_methods"): Str(),
Optional("enable_for_virtual"): Bool(),
}),
"woo/woocommerce_checkout_privacy_policy_text":
Str(),
"woo/woocommerce_registration_privacy_policy_text":
Str(),
".woo/woocommerce_bacs_settings_format":
Enum(["serialized"]),
".woo/woocommerce_cod_settings_format":
Enum(["serialized"]),
Optional(".woo/woocommerce_tax_classes_format"):
Enum(["implode_newline"]),
}
@staticmethod
def load(path: Path, schema_pointer):
"""Load and validate .yaml file."""
schema = copy.deepcopy(schema_pointer)
with path.open() as f:
yaml = f.read()
data = yaml_load(yaml, Any())
is_template = path.name == "template.yaml"
# Replace real Country and Timezone values with fakes
if is_template:
schema["woo/woocommerce_default_country"] = Enum(["LL"])
schema["wp/timezone_string"] = Enum(["Region/Country"])
schema["wp/DEFAULT_WPLANG"] = Enum(["ll_LL"])
schema["woo/woocommerce_currency"] = Enum(["LLL"])
if "woo/woocommerce_tax_classes" in data:
# Inspect that tax classes and taxes match
# create enum for taxes from defined tax_classes
tax_classes = [
str(tax).lower().replace(" ", "-")
for tax in data["woo/woocommerce_tax_classes"]
]
# +1 is for standard schema which is never defined in tax class
for x in range(len(tax_classes) + 1):
# start counting with 1
schema[f"wootax/{x+1}"] = Map({
"country":
Enum(["LL"]) if is_template else Enum(COUNTRIES),
"state":
Str(),
"rate":
Decimal(),
"name":
Str(),
"priority":
Int(),
"compound":
Int(),
"shipping":
Int(),
"order":
Int(),
"class":
Enum([""]) if x == 0 else Enum(tax_classes),
"locations":
Map({}),
})
try:
return yaml_load(yaml, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
@staticmethod
def load_string(data: bytes, schema, path: str):
"""Load and validate yaml data."""
try:
return yaml_load(data, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
python_version=GivenProperty(Str()),
selenium_version=GivenProperty(Str()),
website=GivenProperty(MapPattern(Str(), Str())),
selectors_yml=GivenProperty(Str()),
javascript=GivenProperty(Str()),
setup=GivenProperty(Str()),
code=GivenProperty(Str()),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
)
def __init__(self, keypath, settings):
self.path = keypath
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
self.path.profile = self.path.gen.joinpath("profile")
dirtemplate.DirTemplate(
"webapp", self.path.key / "htmltemplate", self.path.state
).with_vars(javascript=self.given.get("javascript", "")).with_files(
base_html={
filename: {
"content": content
}
for filename, content in self.given.get("website", {}).items()
}).ensure_built()
self.path.state.joinpath("selectors.yml").write_text(
self.given["selectors.yml"])
self.server = (python("-m", "http.server").in_dir(self.path.state /
"webapp").pexpect())
self.server.expect("Serving HTTP on 0.0.0.0")
if not self.path.profile.exists():
self.path.profile.mkdir()
self.python = project_build(self.path, self.given["python version"],
self.given["selenium version"]).bin.python
self.example_py_code = (ExamplePythonCode(
self.python,
self.path.state).with_setup_code(self.given.get(
"setup", "")).with_terminal_size(160, 100).with_long_strings())
@validate(
code=Str(),
will_output=Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
raises=Map({
Optional("type"):
Map({
"in python 2": Str(),
"in python 3": Str()
})
| Str(),
Optional("message"):
Map({
"in python 2": Str(),
"in python 3": Str()
})
| Str(),
}),
in_interpreter=Bool(),
)
def run(self, code, will_output=None, raises=None, in_interpreter=False):
if in_interpreter:
code = "{0}\nprint(repr({1}))".format(
"\n".join(code.strip().split("\n")[:-1]),
code.strip().split("\n")[-1])
to_run = self.example_py_code.with_code(code)
if self.settings.get("cprofile"):
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug)))
result = (to_run.expect_exceptions().run()
if raises is not None else to_run.run())
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")])
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self.settings.get("rewrite"):
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2") else
exception_type["in python 3"])
if message is not None:
if not isinstance(message, str):
differential = True
message = (message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"])
try:
result.exception_was_raised(exception_type, message)
except ExpectedExceptionMessageWasDifferent:
if self.settings.get("rewrite") and not differential:
new_raises = raises.copy()
new_raises["message"] = result.exception.message
self.current_step.update(raises=new_raises)
else:
raise
def do_nothing(self):
pass
def pause(self, message="Pause"):
import IPython
IPython.embed()
def tear_down(self):
self.server.kill(signal.SIGTERM)
self.server.wait()
SHARED_FIELDS_KEYS = {
"install_msg": Str(),
"author": Seq(Str()),
**COMMON_KEYS,
}
#: `cogs` metadata keys.
COG_KEYS = {
"name": Str(), # Downloader doesn't use this but I can set friendlier name
"short": Str(),
"description": Str(),
"end_user_data_statement": Str(),
Optional("class_docstring"): Str(),
Optional("install_msg"): Str(),
Optional("author"): Seq(Str()),
Optional("required_cogs", {}): EmptyDict() | MapPattern(Str(), Url()),
Optional("requirements", []): EmptyList() | Seq(Str()),
Optional("tags", []): EmptyList() | Seq(Str()),
**COMMON_KEYS,
}
#: Root schema of the info.yaml file.
SCHEMA = Map({
"repo": Map(REPO_KEYS),
"shared_fields": Map(SHARED_FIELDS_KEYS),
"cogs": MapPattern(Str(), Map(COG_KEYS)),
})
#: Keys that should be skipped when outputting to info.json.
#: These keys are for infogen's usage.
KEYS_TO_SKIP_IN_COG_INFO = {"class_docstring"}
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
files=GivenProperty(MapPattern(Str(), Str())),
python_version=GivenProperty(Str()),
setup=GivenProperty(Str()),
)
info_definition = InfoDefinition(
docs=InfoProperty(schema=Str()),
)
def __init__(self, paths, settings):
self.path = paths
self.settings = settings
self._build = hitchpylibrarytoolkit.PyLibraryBuild(
"hitchqs",
self.path,
)
def set_up(self):
"""Set up the environment ready to run the stories."""
self._build.ensure_built()
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree()
self.path.state.mkdir()
for filename, content in self.given['files'].items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(content)
self.python = self._build.bin.python
def _process_exception(self, string):
return string.replace(self.path.state, "/path/to")
@no_stacktrace_for(AssertionError)
@no_stacktrace_for(HitchRunPyException)
@validate(
code=Str(),
will_output=Str(),
raises=Map({
Optional("type"): Str(),
Optional("message"): Str(),
})
)
def run(self, code, will_output=None, raises=None):
self.example_py_code = ExamplePythonCode(self.python, self.path.state)\
.with_terminal_size(160, 100)\
.with_setup_code(self.given['setup'])\
.in_dir(self.path.state)
to_run = self.example_py_code.with_code(code)
if self.settings.get("cprofile"):
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug))
)
result = to_run.expect_exceptions().run() if raises is not None else to_run.run()
actual_output = result.output
if will_output is not None:
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self.settings.get("overwrite artefacts"):
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
exception_type = raises.get('type')
message = raises.get('message')
try:
result.exception_was_raised(exception_type)
exception_message = self._process_exception(result.exception.message)
Templex(message).assert_match(exception_message)
except AssertionError:
if self.settings.get("overwrite artefacts"):
new_raises = raises.copy()
new_raises['message'] = exception_message
self.current_step.update(raises=new_raises)
else:
raise
@no_stacktrace_for(AssertionError)
@validate(files=MapPattern(Str(), Str()))
def build_output_is(self, files):
for filename, content in files.items():
filepath = self.path.state.joinpath(filename)
assert filepath.exists(), "{0} does not exist".format(filename)
if content != "":
try:
Templex(content).assert_match(filepath.text())
except AssertionError as error:
raise AssertionError("{0} is nonmatching:\n\n{1}".format(filename, error))
actual_files = [
path.replace(self.path.state + "/", "")
for path in pathquery(self.path.state.joinpath("example")).is_not_dir()
]
assert len(actual_files) == len(files.keys()), \
"Should be:\n\n{0}\n\nAre actually:\n\n{1}\n".format(
'\n'.join(files.keys()),
'\n'.join(actual_files),
)
def on_success(self):
self.new_story.save()
MapPattern(
Str(
doc={
"text":
"The name of the output, as it will be referenced through the client library from within the function implementation."
}),
Map(
{
"target":
Str(
doc={
"text":
"The name of the resource that will receive or be triggered by the messages sent over this output"
}),
Optional("transform"):
Any(doc={"any_options": ["transform"]}),
Optional("filter"):
Any(doc={"any_options": ["filter"]}),
},
doc={
"title":
"output",
"text":
"The target resource and, optionally, any transforms or filters that are applied before it is invoked."
},
),
doc={
"title":
"outputs",
"text":
"The (named) outputs of the function, which will receive or be triggered by the messages that are emitted by the function. The marshaling and connectivity will all be handled by the Plausible client library and the deployment",
},
),
def build(self):
if self._build_path.exists():
self._build_path.rmtree()
self._build_path.mkdir()
if self._src_path.joinpath("dirtemplate.yml").exists():
config = load(
self._src_path.joinpath("dirtemplate.yml").text(),
Map({
Optional("base templates"):
Str(),
"templated":
Seq(
MapPattern(
Str(),
Map({
Optional("content"): Bool(),
Optional("filename"): Bool(),
}))),
})).data
else:
config = {"templated": []}
src_paths = list(pathquery(self._src_path))
templated_filenames = [
list(template.keys())[0] for template in config['templated']
]
if "base templates" in config:
templated_filenames.extend(
pathquery(self._src_path.joinpath(config['base templates'])))
non_templated = []
for srcpath in src_paths:
relpath = srcpath.relpath(self._src_path)
add = True
if relpath in templated_filenames:
add = False
if relpath == "dirtemplate.yml":
add = False
if srcpath.isdir():
add = False
if "base templates" in config:
if relpath.startswith(config['base templates']):
add = False
if add:
non_templated.append(relpath)
for relpath in non_templated:
dest_path = self._dest.joinpath(relpath)
if not dest_path.dirname().exists():
dest_path.dirname().makedirs()
if str(relpath) not in self._ignore_files:
self._src_path.joinpath(relpath).copy(dest_path)
for template_configuration in config['templated']:
for src_path in src_paths:
if not src_path.isdir():
relpath = src_path.relpath(self._src_path)
if relpath in template_configuration.keys():
if 'filename' in template_configuration[relpath]:
slug = slugify(relpath, separator=u'_')
if slug in self._files.keys():
for filename, variables in self._files[
slug].items():
dest_path = self._dest.joinpath(filename)
if not dest_path.dirname().exists():
dest_path.dirname().makedirs()
render_vars = {}
for name, var in self._render_vars.items():
render_vars[name] = var
for name, filevar in variables.items():
render_vars[name] = filevar
render_vars['thisdir'] = pathquery(
dest_path.dirname())
dest_path.write_text(
render(
src_path, self._functions,
render_vars,
base_templates(
self._src_path,
config.get("base templates"))))
else:
raise Exception(
("{0} templated filename exists but not "
"specified with with_files".format(
relpath)))
else:
dest_path = self._dest.joinpath(relpath)
if not dest_path.dirname().exists():
dest_path.dirname().makedirs()
render_vars = copy(self._render_vars)
render_vars['thisdir'] = pathquery(
dest_path.dirname())
if template_configuration[relpath]['content']:
dest_path.write_text(
render(
src_path,
self._functions,
render_vars,
base_templates(
self._src_path,
config.get("base templates")),
))
self.refingerprint()
from strictyaml import Any, Map, MapPattern, Optional, Str
addon = Map({
"spec": Str(),
Optional("install"): MapPattern(Str(), Any(), minimum_keys=1),
"meta": Map({
"name": Str(),
"version": Str()
}),
Optional("config"): MapPattern(Str(), Any(), minimum_keys=1),
})
application = Map({
"spec":
Str(),
Optional("meta"):
Map({
Optional("name"): Str(),
Optional("version"): Str()
}),
Optional("services"):
MapPattern(Str(), Map({"type": Str()})),
Optional("addons"):
MapPattern(
Str(),
Map({
"manager": Str(),
"hash": Str(),
Optional("settings"): MapPattern(Str(), Any(), minimum_keys=1),
}),
minimum_keys=1,
def __init__(self) -> None:
from strictyaml import Map, MapPattern, Optional, Str, Seq, Int, Bool, EmptyList, OrValidator
d_named_object = {
'name': Str(),
'description': Str(),
}
d_feture_check = {
Optional('since', default=''): Str(),
Optional('deprecated', default=''): Str(),
}
self.s_posarg = Map({
**d_feture_check,
'description': Str(),
'type': Str(),
Optional('default', default=''): Str(),
})
self.s_varargs = Map({
**d_named_object,
**d_feture_check,
'type': Str(),
Optional('min_varargs', default=-1): Int(),
Optional('max_varargs', default=-1): Int(),
})
self.s_kwarg = Map({
**d_feture_check,
'type': Str(),
'description': Str(),
Optional('required', default=False): Bool(),
Optional('default', default=''): Str(),
})
self.s_function = Map({
**d_named_object,
**d_feture_check,
'returns':
Str(),
Optional('notes', default=[]):
OrValidator(Seq(Str()), EmptyList()),
Optional('warnings', default=[]):
OrValidator(Seq(Str()), EmptyList()),
Optional('example', default=''):
Str(),
Optional('posargs'):
MapPattern(Str(), self.s_posarg),
Optional('optargs'):
MapPattern(Str(), self.s_posarg),
Optional('varargs'):
self.s_varargs,
Optional('posargs_inherit', default=''):
Str(),
Optional('optargs_inherit', default=''):
Str(),
Optional('varargs_inherit', default=''):
Str(),
Optional('kwargs'):
MapPattern(Str(), self.s_kwarg),
Optional('kwargs_inherit', default=[]):
OrValidator(OrValidator(Seq(Str()), EmptyList()), Str()),
})
self.s_object = Map({
**d_named_object,
**d_feture_check, 'long_name':
Str(),
Optional('extends', default=''):
Str(),
Optional('notes', default=[]):
OrValidator(Seq(Str()), EmptyList()),
Optional('warnings', default=[]):
OrValidator(Seq(Str()), EmptyList()),
Optional('example', default=''):
Str(),
Optional('methods'):
Seq(self.s_function),
Optional('is_container', default=False):
Bool()
})
#!/usr/bin/python3
import os
import sys
cur_dir = os.path.dirname(os.path.abspath(__file__))
# subdir = 'strictyaml-1.0.6'
subdir = 'strictyaml'
sys.path.append(os.path.join(cur_dir, subdir))
# Depende de ruamel: python3-ruamel.yaml
# Depende de dateutil: python3-dateutil
from strictyaml import (load, Map, Str, Int, Seq, Any, Bool, Optional,
MapPattern, YAMLError)
from strictyaml.exceptions import InconsistentIndentationDisallowed
schema_ver = MapPattern(Str(), Any())
# fname = 'scanner_error.yaml'
# fname = 'indent.yaml'
fname = 'test.yaml'
with open(fname) as f:
s = f.read()
try:
parsed = load(s, schema_ver, label=fname)
except InconsistentIndentationDisallowed as e:
print('Use the same indentation across the file')
print(e)
sys.exit(1)
except YAMLError as e:
print('YAML parsing error:')
print(e)
sys.exit(1)
schema = Map({
class Engine(BaseEngine):
"""Engine for running tests and inspecting result."""
given_definition = GivenDefinition(
files=GivenProperty(MapPattern(Str(), Str())),
setup=GivenProperty(Str()),
)
def __init__(self, paths, rewrite, debug):
self.path = paths
self._rewrite = rewrite
self._debug = debug
def set_up(self):
self.path.state = self.path.gen.joinpath("state")
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
self.path.example = self.path.project / "example"
if self.path.example.exists():
self.path.example.rmtree()
self.path.example.makedirs()
for filename, text in self.given.get("files", {}).items():
filepath = self.path.example.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(text)
ubuntu = hitchbuildvagrant.Box("hitchkey", "ubuntu-trusty-64")\
.with_download_path(self.path.share)\
.which_syncs(self.path.project, "/hitchkey")
setup_code = self.given.get("setup", "")
class PythonSnapshot(hitchbuildvagrant.Snapshot):
def setup(self):
self.cmd(setup_code).run()
self.cmd("sudo apt-get install python-setuptools -y").run()
self.cmd("sudo apt-get install build-essential -y").run()
self.cmd("sudo apt-get install python-pip -y").run()
self.cmd("sudo apt-get install python-virtualenv -y").run()
self.cmd("sudo apt-get install python3 -y").run()
self.cmd("cd /hitchkey/ ; sudo pip install .").run()
setuphash = md5(setup_code.encode('utf8')).hexdigest()
self.snapshot = PythonSnapshot("hitchkey_{}".format(setuphash),
ubuntu).with_build_path(self.path.gen)
self.snapshot.ensure_built()
self.snapshot.box.vagrant("rsync").run()
self.snapshot.cmd(
"cd /hitchkey/ ; sudo pip uninstall hitchkey -y ; sudo pip install ."
).run()
@validate(timeout=Int(), exit_code=Int())
@no_stacktrace_for(ICommandError)
@no_stacktrace_for(AssertionError)
def run(self, cmd=None, will_output=None, timeout=240, exit_code=0):
process = self.snapshot.cmd(cmd.replace("\n",
" ; ")).interact().screensize(
160, 100).run()
process.wait_for_finish(timeout=timeout)
actual_output = '\n'.join(process.stripshot().split("\n")[:-1])
if will_output is not None:
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(**{"will output": actual_output})
else:
raise
assert process.exit_code == exit_code, "Exit code should be {} was {}, output:\n{}".format(
exit_code,
process.exit_code,
actual_output,
)
def pause(self):
import IPython
IPython.embed()
def on_failure(self, reason):
if self._debug:
import IPython
IPython.embed()
def tear_down(self):
"""Clean out the state directory."""
if hasattr(self, 'snapshot'):
self.snapshot.shutdown()
from strictyaml import Any, Enum, Int, Map, MapPattern, Optional, Regex, Seq, Str
limits_schema = Map(
{
"time": Int(),
Optional("memory", default=8192): Int(),
Optional("output"): Int(),
Optional("cores"): Int(),
}
)
module_schema = Regex(r"\.?\w+(\.\w+)*")
plugin_schema = Map(
{"module": module_schema, Optional("config"): MapPattern(Str(), Any())}
)
task_sources = Enum(["local", "url"])
schema = Map(
{
"title": Str(),
Optional("description"): Str(),
"limits": limits_schema,
"steps": Map(
{"run": Seq(plugin_schema), Optional("analysis"): Seq(plugin_schema)}
),
"observers": Seq(plugin_schema),
"tasks": MapPattern(Str(), MapPattern(Str(), Any())),
"tools": MapPattern(
Str(),
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
python_version=GivenProperty(Str()),
files=GivenProperty(MapPattern(Str(), Str())),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
fails_on_python_2=InfoProperty(schema=Bool()),
description=InfoProperty(schema=Str()),
experimental=InfoProperty(schema=Bool()),
)
def __init__(self, keypath, rewrite=False, build=False):
self.path = keypath
self._rewrite = rewrite
self._build = hitchpylibrarytoolkit.PyLibraryBuild(
"hitchqs",
self.path,
)
def set_up(self):
"""Set up your applications and the test environment."""
self._build.ensure_built()
self.path.state = self.path.project.parent / "tempqs"
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
for filename, contents in self.given.get("files", {}).items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(contents)
self.path.profile = self.path.gen.joinpath("profile")
if not self.path.profile.exists():
self.path.profile.mkdir()
self.qs = self._build.bin.quickstart
def _run(self, command, args, will_output=None, exit_code=0, timeout=5):
process = command(*shlex.split(args)).interact().screensize(160, 80).run()
process.wait_for_finish()
actual_output = process.stripshot()\
.replace(self.path.state, "/path/to")
# Replace 35.1 SECONDS with n.n SECONDS
actual_output = re.sub("[0-9]+\.[0-9]", "n.n", actual_output)
if will_output is not None:
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(**{"will output": actual_output})
else:
raise
assert process.exit_code == exit_code, "Exit code should be {} was {}, output:\n{}".format(
exit_code,
process.exit_code,
actual_output,
)
@validate(timeout=Int(), exit_code=Int())
@no_stacktrace_for(AssertionError)
def quickstart(self, args, will_output=None, exit_code=0, timeout=5):
self._run(self.qs.in_dir(self.path.state), args, will_output, exit_code, timeout=timeout)
@no_stacktrace_for(AssertionError)
def hk(self, args, will_output=None, exit_code=0, timeout=5, in_dir=""):
if self._build:
self._run(Command("hk").in_dir(self.path.state / in_dir), args, will_output, exit_code, timeout=timeout)
def initial_hk(self, args="", in_dir=""):
if self._build:
Command("hk", *shlex.split(args)).in_dir(self.path.state / in_dir).run()
@validate(filenames=Seq(Str()))
def files_appear(self, filenames):
appeared = set()
should_appear = set(filenames)
for existing_file in pathquery(self.path.state):
if "__pycache__" not in existing_file and not existing_file.isdir():
appeared.add(str(existing_file.relpath(self.path.state)))
diff = should_appear.symmetric_difference(appeared)
assert diff == set(), \
"Difference in files that appeared:\n{}".format('\n'.join(diff))
def pause(self, message="Pause"):
import IPython
IPython.embed()
def tear_down(self):
if self._build:
if self.path.state.exists():
Command("hk", "--clean").ignore_errors().in_dir(self.path.state).run()
class Experiment:
"""
This class orchestrates the analysis pipeline for our redox imaging experiments.
"""
experiment_schema = Map({
"pipeline":
Map({
"strategy": Str(),
"acquisition_method": Enum(["acquire", "mda"]),
"trimmed_profile_length": Int(),
"untrimmed_profile_length": Int(),
"seg_threshold": Int(),
"measurement_order": Int(),
"measure_thickness": Float(),
"reference_wavelength": Str(),
"image_register": Int(),
"channel_register": Int(),
"population_register": Int(),
"trimmed_regions": MapPattern(Str(), CommaSeparated(Float())),
"untrimmed_regions": MapPattern(Str(), CommaSeparated(Float())),
}),
"redox":
Map({
"ratio_numerator": Str(),
"ratio_denominator": Str(),
"r_min": Float(),
"r_max": Float(),
"instrument_factor": Float(),
"midpoint_potential": Float(),
"z": Int(),
"temperature": Float(),
}),
"registration":
Map({
"n_deriv": Float(),
"warp_n_basis": Float(),
"warp_order": Float(),
"warp_lambda": Float(),
"smooth_lambda": Float(),
"smooth_n_breaks": Float(),
"smooth_order": Float(),
"rough_lambda": Float(),
"rough_n_breaks": Float(),
"rough_order": Float(),
}),
"output":
Map({
"should_save_plots": Bool(),
"should_save_profile_data": Bool(),
"should_save_summary_data": Bool(),
}),
})
seg_images: xr.DataArray = None
rot_fl: xr.DataArray = None
rot_seg: xr.DataArray = None
midlines: xr.DataArray = None
untrimmed_raw_profiles: xr.DataArray = None
untrimmed_std_profiles: xr.DataArray = None
untrimmed_reg_profiles: xr.DataArray = None
trimmed_raw_profiles: xr.DataArray = None
trimmed_std_profiles: xr.DataArray = None
trimmed_reg_profiles: xr.DataArray = None
channel_warps: xr.DataArray = None
std_warps: xr.DataArray = None
def __init__(self, exp_dir):
self.experiment_dir = Path(exp_dir)
self.settings_path = self.experiment_dir.joinpath("settings.yaml")
try:
with open(self.settings_path, "r") as f:
self.config = load(f.read(), self.experiment_schema).data
except YAMLError:
raise ValueError("Incorrectly specified config file.")
self.experiment_id = self.experiment_dir.stem
# compute the filenames/paths for this experiment
self.movement_path = self.experiment_dir.joinpath(self.experiment_id +
"-mvmt.csv")
self.frame_map_path = self.experiment_dir.joinpath(self.experiment_id +
"-frame_map.csv")
self.processed_images_dir = self.experiment_dir.joinpath(
"processed_images")
self.rot_seg_dir = self.processed_images_dir.joinpath("rot_seg")
self.rot_fl_dir = self.processed_images_dir.joinpath("rot_fl")
self.fl_imgs_dir = self.processed_images_dir.joinpath(
"fluorescent_images")
self.orig_images_path = self.processed_images_dir.joinpath("images.nc")
self.seg_images_path = self.processed_images_dir.joinpath(
"seg_images.nc")
self.aligned_images_path = self.processed_images_dir.joinpath(
"aligned_images.nc")
self.aligned_seg_images_path = self.processed_images_dir.joinpath(
"aligned_seg_images.nc")
# load images
self.images = self._load_raw_images()
# try to load masks
try:
self.load_masks()
except IOError:
logging.info("No masks found in experiment directory")
pass
# Computed Filepaths
@property
def midlines_path(self) -> Path:
return self.analysis_dir.joinpath("midlines.pickle")
@property
def raw_img_stack_path(self) -> Path:
# TODO test that this works
accepted_extensions = [".tif", ".tiff", ".stk"]
candidate_paths = [
self.experiment_dir.joinpath(f"{self.experiment_id}{ext}")
for ext in accepted_extensions
]
for path in candidate_paths:
if path.exists():
return path
raise ValueError(
f"No image found in experiment directory. Tried the following files: {candidate_paths}"
)
@property
def fig_dir(self):
return self.analysis_dir.joinpath("figs")
def untrimmed_profile_data_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-untrimmed_{treatment}_profile_data.nc")
def trimmed_profile_data_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-trimmed_{treatment}_profile_data.nc")
@property
def channel_warp_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-channel_warps.nc")
@property
def std_warp_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id + "-std_warps.nc")
def untrimmed_profile_data_csv_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-untrimmed_{treatment}_profile_data.csv")
def trimmed_profile_data_csv_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-trimmed_{treatment}_profile_data.csv")
@property
def untrimmed_region_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-untrimmed_region_data.csv")
@property
def trimmed_region_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-trimmed_region_data.csv")
@property
def analysis_dir(self) -> Path:
date_str = datetime.datetime.now().strftime("%Y-%m-%d")
strategy = self.config["pipeline"]["strategy"]
if len(strategy) > 0:
suffix = f"_{strategy}"
else:
suffix = ""
analysis_dir_ = self.experiment_dir.joinpath(
"analyses",
utils.get_valid_filename(f"{date_str}{suffix}"),
)
# analysis_dir_.mkdir(parents=True, exist_ok=True)
return analysis_dir_
def _load_raw_images(self):
"""
This returns the raw (non-median-subtracted) images
"""
logging.info(f"Loading image data from {self.raw_img_stack_path}")
raw_image_data = pio.load_tiff_as_hyperstack(
img_stack_path=self.raw_img_stack_path,
manual_metadata=self.frame_map_path,
mvmt_metadata=self.movement_path,
)
raw_image_data = raw_image_data.assign_coords({
"experiment_id": (
("animal", ),
np.repeat(self.experiment_id, raw_image_data.animal.size),
)
})
raw_image_data = self.add_experiment_metadata_to_data_array(
raw_image_data)
return raw_image_data
def _load_movement(self) -> pd.DataFrame:
movement_path = self.experiment_dir.joinpath(self.experiment_id +
"-mvmt.csv")
try:
df = pd.read_csv(movement_path)
df = df.pivot_table(index="animal",
columns=["region", "pair"],
values="movement")
df = df.stack("pair")
return df
except FileNotFoundError:
logging.warning(
f"Tried to access {movement_path}; file was not found")
return None
def make_analysis_dir(self) -> None:
logging.info(f"Making analysis directory at {self.analysis_dir}")
self.analysis_dir.mkdir(parents=True, exist_ok=True)
@property
def trimmed_summary_table(self):
df = profile_processing.summarize_over_regions(
self.trimmed_raw_profiles,
regions=self.config["pipeline"]["trimmed_regions"],
rescale=False,
**self.config["redox"],
)
return df
@property
def untrimmed_summary_table(self):
df = profile_processing.summarize_over_regions(
self.untrimmed_raw_profiles,
regions=self.config["pipeline"]["untrimmed_regions"],
**self.config["redox"],
)
return df
####################################################################################
# PIPELINE
####################################################################################
def full_pipeline(self):
logging.info(f"Starting full pipeline run for {self.experiment_dir}")
self.make_analysis_dir()
logging.info(f"Saving fluorescent images to {self.fl_imgs_dir}")
pio.save_images_xarray_to_tiffs(self.images,
self.fl_imgs_dir,
prefix=self.experiment_id)
self.segment_pharynxes()
self.register_images()
self.align_and_center()
self.calculate_midlines()
self.measure_under_midlines()
self.register_profiles()
self.trim_data()
self.calculate_redox()
self.do_manual_ap_flips()
self.persist_to_disk()
logging.info(f"Finished full pipeline run for {self.experiment_dir}")
return self
def run_neuron_pipeline(self):
logging.info(
f"Starting full neuron analysis pipeline run for {self.experiment_dir}"
)
self.make_analysis_dir()
df = ip.measure_under_labels(self.images,
self.seg_images).reset_index()
df.to_csv(self.analysis_dir /
(self.experiment_id + "-neuron_analysis.csv"))
def segment_pharynxes(self):
if self.seg_images is not None:
logging.info("masks have been specified. skipping mask generation")
self.save_masks()
return
else:
logging.info("Generating masks")
self.seg_images = ip.segment_pharynxes(
self.images,
wvl=self.config["pipeline"]["reference_wavelength"])
self.save_masks()
def register_images(self):
if self.config["pipeline"]["image_register"]:
logging.info("Registering Images")
self.images = ip.register_all_images(self.images, self.seg_images)
def align_and_center(self):
logging.info("Centering and rotating pharynxes")
self.rot_fl, self.rot_seg = ip.center_and_rotate_pharynxes(
self.images,
self.seg_images,
)
logging.info(f"Saving rotated FL images to {self.aligned_images_path}")
pio.save_profile_data(self.rot_fl, self.aligned_images_path)
logging.info(f"Saving rotated masks to {self.aligned_seg_images_path}")
pio.save_profile_data(self.rot_seg, self.aligned_seg_images_path)
def calculate_midlines(self):
logging.info("Calculating midlines")
self.midlines = ip.calculate_midlines(self.rot_seg, degree=4)
def measure_under_midlines(self):
logging.info("Measuring under midlines")
self.untrimmed_raw_profiles = ip.measure_under_midlines(
self.rot_fl,
self.midlines,
n_points=self.config["pipeline"]["untrimmed_profile_length"],
order=self.config["pipeline"]["measurement_order"],
thickness=float(self.config["pipeline"]["measure_thickness"]),
)
self.untrimmed_raw_profiles = profile_processing.align_pa(
self.untrimmed_raw_profiles)
self.untrimmed_raw_profiles = self.add_experiment_metadata_to_data_array(
self.untrimmed_raw_profiles)
# subtract the image medians from the profile data
logging.info("Subtracting image medians from profile data")
self.untrimmed_raw_profiles = ip.subtract_medians(
self.untrimmed_raw_profiles, self.images)
def register_profiles(self):
if self.config["pipeline"]["population_register"]:
logging.info("Standardizing profiles")
(
self.untrimmed_std_profiles,
self.std_warps,
) = profile_processing.standardize_profiles(
self.untrimmed_raw_profiles,
redox_params=self.config["redox"],
**self.config["registration"],
)
if self.config["pipeline"]["channel_register"]:
logging.info("Channel-Registering profiles")
if self.untrimmed_std_profiles is not None:
logging.info(
"using the standardize profiles for channel-registration")
data_to_register = self.untrimmed_std_profiles
else:
logging.info("using the raw profiles for channel-registration")
data_to_register = self.untrimmed_raw_profiles
(
self.untrimmed_reg_profiles,
self.channel_warps,
) = profile_processing.channel_register(
data_to_register,
redox_params=self.config["redox"],
reg_params=self.config["registration"],
)
def trim_data(self):
logging.info("Trimming intensity data")
self.trimmed_raw_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_raw_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
if self.untrimmed_std_profiles is not None:
self.trimmed_std_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_std_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
if self.untrimmed_reg_profiles is not None:
self.trimmed_reg_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_reg_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
def calculate_redox(self):
logging.info("Calculating redox measurements")
redox_params = self.config["redox"]
# Images
self.images = utils.add_derived_wavelengths(self.images,
**redox_params)
self.rot_fl = utils.add_derived_wavelengths(self.rot_fl,
**redox_params)
# profiles
self.trimmed_raw_profiles = utils.add_derived_wavelengths(
self.trimmed_raw_profiles, **redox_params)
self.untrimmed_raw_profiles = utils.add_derived_wavelengths(
self.untrimmed_raw_profiles, **redox_params)
def do_manual_ap_flips(self):
# TODO finish implementation
logging.info("skipping manual AP flips - not implemented")
def flip_at(self, idx):
# TODO finish implementation
raise NotImplementedError
####################################################################################
# PERSISTENCE / IO
####################################################################################
def save_images(self):
"""Save this experiment's images to disk as netCDF4 files"""
imgs_paths = [
(self.images, self.orig_images_path),
(self.rot_fl, self.aligned_images_path),
(self.seg_images, self.seg_images_path),
(self.rot_seg, self.aligned_seg_images_path),
]
for img, path in imgs_paths:
if img is not None:
logging.info(f"Saving images to {path}")
img.to_netcdf(path)
# def load_tiff_as_hyperstack(self):
# pass
def make_fig_dir(self):
fig_dir = self.analysis_dir.joinpath("figs")
fig_dir.mkdir(parents=True, exist_ok=True)
return fig_dir
def save_individual_profiles(self, profile_data, treatment: str,
trimmed: bool):
if profile_data is None:
return
fig_dir = self.make_fig_dir()
profile_data_fig_dir = (fig_dir / "profile_data" / treatment /
("trimmed" if trimmed else "untrimmed"))
individual_data_fig_dir = profile_data_fig_dir.joinpath("inividual")
individual_data_fig_dir.mkdir(exist_ok=True, parents=True)
for title, fig in plots.generate_wvl_pair_timepoint_profile_plots(
profile_data):
title = title.replace(" ", "")
fig.savefig(individual_data_fig_dir /
f"{self.experiment_id}-{title}-individuals.pdf")
plt.close(fig)
def save_avg_profiles(self, profile_data, treatment: str, trimmed: bool):
if profile_data is None:
return
fig_dir = self.make_fig_dir()
profile_data_fig_dir = (fig_dir / "profile_data" / treatment /
("trimmed" if trimmed else "untrimmed"))
individual_data_fig_dir = profile_data_fig_dir.joinpath("avg")
individual_data_fig_dir.mkdir(exist_ok=True, parents=True)
for title, fig in plots.generate_avg_wvl_pair_profile_plots(
profile_data):
title = title.replace(" ", "")
fig.savefig(individual_data_fig_dir /
f"{self.experiment_id}-{title}-avg.pdf")
plt.close(fig)
def save_plots(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
for data, treatment, trimmed in [
(self.untrimmed_raw_profiles, "raw", False),
(self.untrimmed_std_profiles, "standardized", False),
(self.untrimmed_reg_profiles, "channel-registered", False),
(self.trimmed_raw_profiles, "raw", True),
(self.trimmed_std_profiles, "standardized", True),
(self.trimmed_reg_profiles, "channel-registered", True),
]:
self.save_individual_profiles(data, treatment, trimmed)
self.save_avg_profiles(data, treatment, trimmed)
# frame-normed Ratio Images
mvmt_annotation_img_path = self.fig_dir.joinpath(
f"{self.experiment_id}-movement_annotation_imgs.pdf")
imgs = utils.add_derived_wavelengths(self.images,
**self.config["redox"])
with PdfPages(mvmt_annotation_img_path) as pdf:
for i in tqdm(range(self.images.animal.size)):
fig = plots.plot_pharynx_R_imgs(imgs[i],
mask=self.seg_images[i])
fig.suptitle(f"animal = {i}")
pdf.savefig(fig)
if (i % 20) == 0:
plt.close("all")
# Pop-normed ratio images
u = self.trimmed_raw_profiles.sel(wavelength="r").mean()
std = self.trimmed_raw_profiles.sel(wavelength="r").std()
for pair in self.rot_fl.pair.values:
for tp in self.rot_fl.timepoint.values:
ratio_img_path = self.fig_dir.joinpath(
f"{self.experiment_id}-ratio_images-pair={pair};timepoint={tp}.pdf"
)
with PdfPages(ratio_img_path) as pdf:
logging.info(
f"Saving ratio images to {ratio_img_path}")
for i in tqdm(range(self.rot_fl.animal.size)):
fig, ax = plt.subplots(dpi=300)
ratio_img = (self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_numerator"],
pair=pair,
timepoint=tp,
) / self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_denominator"],
pair=pair,
timepoint=tp,
))[i]
fl_img = self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_numerator"],
pair=pair,
timepoint=tp,
)[i]
im, cbar = plots.imshow_ratio_normed(
ratio_img,
fl_img,
r_min=u - (std * 1.96),
r_max=u + (std * 1.96),
colorbar=True,
i_max=5000,
i_min=1000,
ax=ax,
)
ax.plot(
*self.midlines.sel(
pair=pair,
timepoint=tp,
)[i].values[()].linspace(),
color="green",
alpha=0.3,
)
strain = self.rot_fl.strain.values[i]
ax.set_title(
f"Animal={i} ; Pair={pair} ; Strain={strain}")
cax = cbar.ax
for j in range(len(self.trimmed_raw_profiles)):
cax.axhline(
self.trimmed_raw_profiles.sel(
wavelength="r",
pair=pair,
timepoint=tp)[j].mean(),
color="k",
alpha=0.1,
)
cax.axhline(
self.trimmed_raw_profiles.sel(
wavelength="r", pair=pair,
timepoint=tp)[i].mean(),
color="k",
)
pdf.savefig()
if (i % 20) == 0:
plt.close("all")
def persist_profile_data(self):
for treatment, untrimmed_profile_data in (
("raw", self.untrimmed_raw_profiles),
("std", self.untrimmed_std_profiles),
("reg", self.untrimmed_reg_profiles),
):
if untrimmed_profile_data is not None:
untrimmed_prof_path = self.untrimmed_profile_data_path(
treatment)
logging.info(
f"Saving untrimmed {treatment} profile data to {untrimmed_prof_path}"
)
pio.save_profile_data(untrimmed_profile_data,
untrimmed_prof_path)
untrimmed_prof_path_csv = self.untrimmed_profile_data_csv_path(
treatment)
profile_processing.to_dataframe(
untrimmed_profile_data,
"value").to_csv(untrimmed_prof_path_csv)
for treatment, trimmed_profile_data in (
("raw", self.trimmed_raw_profiles),
("std", self.trimmed_std_profiles),
("reg", self.trimmed_reg_profiles),
):
if trimmed_profile_data is not None:
trimmed_prof_path = self.trimmed_profile_data_path(treatment)
logging.info(
f"Saving trimmed {treatment} profile data to {trimmed_prof_path}"
)
pio.save_profile_data(trimmed_profile_data, trimmed_prof_path)
trimmed_prof_path_csv = self.trimmed_profile_data_csv_path(
treatment)
logging.info(
f"Saving trimmed {treatment} profile data to {trimmed_prof_path_csv}"
)
profile_processing.to_dataframe(
trimmed_profile_data,
"value").to_csv(trimmed_prof_path_csv)
# Warps, if necessary
if self.config["pipeline"]["channel_register"]:
logging.info(
f"Saving channel warp data to {self.channel_warp_data_path}")
self.channel_warps.to_netcdf(self.channel_warp_data_path)
if self.config["pipeline"]["population_register"]:
logging.info(
f"Saving channel warp data to {self.std_warp_data_path}")
self.std_warps.to_netcdf(self.std_warp_data_path)
def save_summary_data(self):
# Persist the region means
logging.info(
f"Saving untrimmed region means to {self.untrimmed_region_data_path}"
)
self.untrimmed_summary_table.to_csv(self.untrimmed_region_data_path)
logging.info(
f"Saving trimmed region means to {self.trimmed_region_data_path}")
self.trimmed_summary_table.to_csv(self.trimmed_region_data_path)
def save_masks(self):
logging.info(f"saving masks to {self.seg_images_path}")
pio.save_profile_data(self.seg_images, self.seg_images_path)
def load_masks(self):
self.seg_images = pio.load_profile_data(self.seg_images_path)
logging.info(f"Loaded masks from {self.seg_images_path}")
def save_midlines(self):
pio.save_midlines(self.midlines_path, self.midlines)
def load_midlines(self):
return pio.load_midlines(self.midlines_path)
def persist_to_disk(self):
logging.info(
f"Saving {self.experiment_id} inside {self.experiment_dir}")
self.save_midlines()
if self.config["output"]["should_save_summary_data"]:
self.save_summary_data()
if self.config["output"]["should_save_profile_data"]:
self.persist_profile_data()
if self.config["output"]["should_save_plots"]:
self.save_plots()
####################################################################################
# MISC / HELPER
####################################################################################
def add_experiment_metadata_to_data_array(self, data_array: xr.DataArray):
params = {}
params.update(self.config["pipeline"])
params.update(self.config["redox"])
params.update(self.config["registration"])
to_remove = ["trimmed_regions", "untrimmed_regions"]
for k in to_remove:
del params[k]
return data_array.assign_attrs(**params)
'executionTimeout': None,
'killTimeout': 30,
'statsd': None,
}
_report_schema = Map({
Opt("sentry"): Map({
Opt("dsn"): Map({
Opt("value"): EmptyNone() | Str(),
Opt("fromFile"): EmptyNone() | Str(),
Opt("fromEnvVar"): EmptyNone() | Str(),
}),
Opt("fingerprint"): Seq(Str()),
Opt("level"): Str(),
Opt("extra"): MapPattern(Str(), Str() | Int() | Bool()),
}),
Opt("mail"): Map({
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
})
})
_job_defaults_common = {
Opt("shell"): Str(),
Opt("concurrencyPolicy"): Enum(['Allow', 'Forbid', 'Replace']),
Opt("captureStderr"): Bool(),
_report_schema = Map({
Opt("sentry"):
Map({
Opt("dsn"):
Map({
Opt("value"): EmptyNone() | Str(),
Opt("fromFile"): EmptyNone() | Str(),
Opt("fromEnvVar"): EmptyNone() | Str(),
}),
Opt("fingerprint"):
Seq(Str()),
Opt("level"):
Str(),
Opt("extra"):
MapPattern(Str(),
Str() | Int() | Bool()),
Opt("body"):
Str(),
}),
Opt("mail"):
Map({
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
}),
})
_job_defaults_common = {
Map({
"name": Str(),
"value": Int() | Float(),
Optional("nominalValue"): Int() | Float(),
Optional("linkedValue"): Int() | Float(),
Optional("rangeMinValue"): Int() | Float(),
Optional("rangeMaxValue"): Int() | Float(),
Optional("flags"): Int()
}))
}), )
stat_format4_schema = Seq(
Map({
"name": Str(),
Optional("flags"): Int(),
"location": MapPattern(Str(),
Int() | Float()),
}))
instance_schema = MapPattern(
Str(),
Seq(
Map({
Optional("familyName"): Str(),
Optional("styleName"): Str(),
"coordinates": MapPattern(Str(),
Int() | Float()),
})))
schema = Map({
"sources":
Seq(Str()),
class Engine(BaseEngine):
"""Python engine for running tests."""
schema = StorySchema(
preconditions=Map({
"files": MapPattern(Str(), Str()),
"variables": MapPattern(Str(), Str()),
"python version": Str(),
"ruamel version": Str(),
}),
params=Map({
"python version": Str(),
"ruamel version": Str(),
}),
about={
"description": Str(),
Optional("importance"): Int(),
},
)
def __init__(self, keypath, settings):
self.path = keypath
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.doc = hitchdoc.Recorder(
hitchdoc.HitchStory(self),
self.path.gen.joinpath('storydb.sqlite'),
)
if self.path.gen.joinpath("state").exists():
self.path.gen.joinpath("state").rmtree(ignore_errors=True)
self.path.gen.joinpath("state").mkdir()
self.path.state = self.path.gen.joinpath("state")
for filename, text in self.preconditions.get("files", {}).items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().mkdir()
filepath.write_text(text)
self.python_package = hitchpython.PythonPackage(
self.preconditions.get('python_version', '3.5.0'))
self.python_package.build()
self.pip = self.python_package.cmd.pip
self.python = self.python_package.cmd.python
# Install debugging packages
with hitchtest.monitor(
[self.path.key.joinpath("debugrequirements.txt")]) as changed:
if changed:
run(
self.pip("install", "-r",
"debugrequirements.txt").in_dir(self.path.key))
# Uninstall and reinstall
run(self.pip("uninstall", "strictyaml", "-y").ignore_errors())
run(self.pip("install", ".").in_dir(self.path.project))
run(
self.pip(
"install", "ruamel.yaml=={0}".format(
self.preconditions["ruamel version"])))
self.services = hitchserve.ServiceBundle(str(self.path.project),
startup_timeout=8.0,
shutdown_timeout=1.0)
self.services['IPython'] = hitchpython.IPythonKernelService(
self.python_package)
self.services.startup(interactive=False)
self.ipython_kernel_filename = self.services[
'IPython'].wait_and_get_ipykernel_filename()
self.ipython_step_library = hitchpython.IPythonStepLibrary()
self.ipython_step_library.startup_connection(
self.ipython_kernel_filename)
self.shutdown_connection = self.ipython_step_library.shutdown_connection
self.ipython_step_library.run("import os")
self.ipython_step_library.run("import sure")
self.ipython_step_library.run("from path import Path")
self.ipython_step_library.run("os.chdir('{}')".format(self.path.state))
for filename, text in self.preconditions.get("files", {}).items():
self.ipython_step_library.run(
"""{} = Path("{}").bytes().decode("utf8")""".format(
filename.replace(".yaml", ""), filename))
def run_command(self, command):
self.ipython_step_library.run(command)
self.doc.step("code", command=command)
def variable(self, name, value):
self.path.state.joinpath("{}.yaml".format(name)).write_text(value)
self.ipython_step_library.run(
"""{} = Path("{}").bytes().decode("utf8")""".format(
name, "{}.yaml".format(name)))
self.doc.step("variable", var_name=name, value=value)
def code(self, command):
self.ipython_step_library.run(command)
self.doc.step("code", command=command)
@validate(exception=Str())
def raises_exception(self, command, exception, why=''):
"""
Command raises exception.
"""
import re
self.error = self.ipython_step_library.run(
command, swallow_exception=True).error
if self.error is None:
raise Exception("Expected exception, but got none")
full_exception = re.compile("(?:\\x1bs?\[0m)+(?:\n+)+{0}".format(
re.escape("\x1b[0;31m"))).split(self.error)[-1]
exception_class_name, exception_text = full_exception.split(
"\x1b[0m: ")
if self.settings.get("overwrite"):
self.current_step.update(exception=str(exception_text))
else:
assert exception.strip(
) in exception_text, "UNEXPECTED:\n{0}".format(exception_text)
self.doc.step(
"exception",
command=command,
exception_class_name=exception_class_name,
exception=exception_text,
why=why,
)
def returns_true(self, command, why=''):
self.ipython_step_library.assert_true(command)
self.doc.step("true", command=command, why=why)
def should_be_equal(self, lhs='', rhs='', why=''):
command = """({0}).should.be.equal({1})""".format(lhs, rhs)
self.ipython_step_library.run(command)
self.doc.step("true", command=command, why=why)
def assert_true(self, command):
self.ipython_step_library.assert_true(command)
self.doc.step("true", command=command)
def assert_exception(self, command, exception):
error = self.ipython_step_library.run(command,
swallow_exception=True).error
assert exception.strip() in error
self.doc.step("exception", command=command, exception=exception)
def on_failure(self):
if self.settings.get("pause_on_failure", True):
if self.preconditions.get("launch_shell", False):
self.services.log(message=self.stacktrace.to_template())
self.shell()
def shell(self):
if hasattr(self, 'services'):
self.services.start_interactive_mode()
import sys
import time
time.sleep(0.5)
if path.exists(
path.join(path.expanduser("~"),
".ipython/profile_default/security/",
self.ipython_kernel_filename)):
call([
sys.executable, "-m", "IPython", "console", "--existing",
"--no-confirm-exit",
path.join(path.expanduser("~"),
".ipython/profile_default/security/",
self.ipython_kernel_filename)
])
else:
call([
sys.executable, "-m", "IPython", "console", "--existing",
self.ipython_kernel_filename
])
self.services.stop_interactive_mode()
def assert_file_contains(self, filename, contents):
assert self.path.state.joinpath(filename).bytes().decode(
'utf8').strip() == contents.strip()
self.doc.step("filename contains",
filename=filename,
contents=contents)
def pause(self, message="Pause"):
if hasattr(self, 'services'):
self.services.start_interactive_mode()
import IPython
IPython.embed()
if hasattr(self, 'services'):
self.services.stop_interactive_mode()
def on_success(self):
if self.settings.get("overwrite"):
self.new_story.save()
def tear_down(self):
try:
self.shutdown_connection()
except:
pass
if hasattr(self, 'services'):
self.services.shutdown()
class Engine(BaseEngine):
"""Python engine for running tests."""
schema = StorySchema(
given={
Optional("files"): MapPattern(Str(), Str()),
Optional("symlinks"): MapPattern(Str(), Str()),
Optional("permissions"): MapPattern(Str(), Str()),
Optional("setup"): Str(),
Optional("python version"): Str(),
Optional("pathpy version"): Str()
},
info={},
)
def __init__(self, keypath, settings):
self.path = keypath
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
if self.path.gen.joinpath("q").exists():
self.path.gen.joinpath("q").remove()
for filename, text in self.given.get("files", {}).items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(text)
for filename, linkto in self.given.get("symlinks", {}).items():
filepath = self.path.state.joinpath(filename)
linktopath = self.path.state.joinpath(linkto)
linktopath.symlink(filepath)
for filename, permission in self.given.get("permissions", {}).items():
filepath = self.path.state.joinpath(filename)
filepath.chmod(int(permission, 8))
pylibrary = hitchbuildpy.PyLibrary(
name="py3.5.0",
base_python=hitchbuildpy.PyenvBuild("3.5.0").with_build_path(
self.path.share),
module_name="pathquery",
library_src=self.path.project,
).with_build_path(self.path.gen)
pylibrary.ensure_built()
self.python = pylibrary.bin.python
self.example_py_code = ExamplePythonCode(self.python, self.path.state)\
.with_code(self.given.get('code', ''))\
.with_setup_code(self.given.get('setup', ''))\
.with_terminal_size(160, 100)\
.with_env(TMPDIR=self.path.gen)\
.with_long_strings(
yaml_snippet_1=self.given.get('yaml_snippet_1'),
yaml_snippet=self.given.get('yaml_snippet'),
yaml_snippet_2=self.given.get('yaml_snippet_2'),
modified_yaml_snippet=self.given.get('modified_yaml_snippet'),
)
@expected_exception(NonMatching)
@expected_exception(HitchRunPyException)
@validate(
code=Str(),
will_output=Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
raises=Map({
Optional("type"):
Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
Optional("message"):
Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
}),
in_interpreter=Bool(),
)
def run(self,
code,
will_output=None,
yaml_output=True,
raises=None,
in_interpreter=False):
if in_interpreter:
code = '{0}\nprint(repr({1}))'.format(
'\n'.join(code.strip().split('\n')[:-1]),
code.strip().split('\n')[-1])
to_run = self.example_py_code.with_code(code)
if self.settings.get("cprofile"):
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug)))
result = to_run.expect_exceptions().run(
) if raises is not None else to_run.run()
if will_output is not None:
actual_output = '\n'.join(
[line.rstrip() for line in result.output.split("\n")])
try:
Templex(will_output).assert_match(actual_output)
except NonMatching:
if self.settings.get("rewrite"):
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get('type')
message = raises.get('message')
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = exception_type['in python 2']\
if self.given['python version'].startswith("2")\
else exception_type['in python 3']
if message is not None:
if not isinstance(message, str):
differential = True
message = message['in python 2']\
if self.given['python version'].startswith("2")\
else message['in python 3']
try:
result = self.example_py_code.expect_exceptions().run()
result.exception_was_raised(exception_type, message)
except ExpectedExceptionMessageWasDifferent as error:
if self.settings.get("rewrite") and not differential:
new_raises = raises.copy()
new_raises['message'] = result.exception.message
self.current_step.update(raises=new_raises)
else:
raise
def output_contains(self, expected_contents, but_not=None):
try:
output_contents = self.path.state.joinpath(
"output.txt").text().strip()
except FileNotFoundError:
raise AssertionError("Output not found")
for expected_item in expected_contents:
found = False
for output_item in output_contents.split('\n'):
if output_item.strip() == str(
self.path.state.joinpath(
expected_item).abspath()).strip():
found = True
if not found:
raise AssertionError(
"Expected:\n{0}\n\nNot found in output:\n{1}".format(
expected_item,
output_contents,
))
if but_not is not None:
for unexpected_item in but_not:
found = False
for output_item in output_contents.split('\n'):
if output_item.strip() == str(
self.path.state.joinpath(
unexpected_item).abspath()).strip():
found = True
if found:
raise RuntimeError(
"Expected NOT to find:\n{0}\n\nBut found in:\n{1}".
format(
unexpected_item,
output_contents,
))
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_success(self):
if self.settings.get("rewrite"):
self.new_story.save()
if self.settings.get("cprofile"):
self.python(
self.path.key.joinpath("printstats.py"),
self.path.profile.joinpath("{0}.dat".format(
self.story.slug))).run()
def tear_down(self):
if self.path.gen.joinpath("q").exists():
print(self.path.gen.joinpath("q").text())
from strictyaml import Map, MapPattern, Optional
from strictyaml import Str, Int, Seq, Enum, Any, as_document
JSONSCHEMA_TYPE_SNIPPET = {
"type": Enum(["object", "integer", "string", "array"]),
Optional("required"): Seq(Str()),
Optional("properties"): MapPattern(Str(), Any()),
Optional("items"): Any(),
}
JSONSCHEMA_SCHEMA = Map(JSONSCHEMA_TYPE_SNIPPET)
def get_schema(snippet):
if snippet['type'] == "integer":
return Int()
elif snippet['type'] == "string":
return Str()
elif snippet['type'] == "array":
return Seq(get_schema(snippet["items"]))
elif snippet['type'] == "object":
map_schema = {}
for key, subschema in snippet['properties'].items():
if key in snippet.get('required', []):
map_schema[Optional(key)] = get_schema(subschema)
else:
map_schema[key] = get_schema(subschema)
return Map(map_schema)
def load_schema(json_schema):
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
setup=GivenProperty(Str()),
boxname=GivenProperty(Str()),
vmname=GivenProperty(Str()),
issue=GivenProperty(Str()),
files=GivenProperty(MapPattern(Str(), Str())),
python_version=GivenProperty(Str()),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
)
def __init__(self, paths, settings):
self.path = paths
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.cachestate = self.path.gen.joinpath("cachestate")
self.path.state = self.path.gen.joinpath("state")
self.path.working_dir = self.path.gen.joinpath("working")
self.path.build_path = self.path.gen.joinpath("build_path")
self.path.localsync = self.path.gen.joinpath("local_sync")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
if self.path.localsync.exists():
self.path.localsync.rmtree(ignore_errors=True)
self.path.localsync.mkdir()
if self.path.build_path.exists():
self.path.build_path.rmtree(ignore_errors=True)
self.path.build_path.mkdir()
self.python = hitchpylibrarytoolkit.project_build(
"hitchbuildvagrant", self.path,
self.given.get("python_version", "3.7.0")).bin.python
if not self.path.cachestate.exists():
self.path.cachestate.mkdir()
for filename, contents in self.given.get("files", {}).items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(contents)
if self.path.working_dir.exists():
self.path.working_dir.rmtree(ignore_errors=True)
self.path.working_dir.mkdir()
self.example_py_code = (ExamplePythonCode(
self.python, self.path.state).with_setup_code(
self.given.get("setup", "").replace(
"/path/to/share",
self.path.cachestate)).with_terminal_size(
160, 100).with_long_strings(
share=str(self.path.cachestate),
build_path=str(self.path.build_path),
issue=str(self.given.get("issue")),
boxname=str(self.given.get("boxname")),
vmname=str(self.given.get("vmname")),
local_sync_path=str(self.path.localsync),
))
@no_stacktrace_for(HitchRunPyException)
def run(self, code):
self.example_py_code.with_code(code).run()
def write_to_localsync(self, **files):
for filename, contents in files.items():
self.path.localsync.joinpath(filename).write_text(contents)
def delete_localsync_file(self, filename):
self.path.localsync.joinpath(filename).remove()
def write_file(self, filename, contents):
self.path.state.joinpath(filename).write_text(contents)
def raises_exception(self, message=None, exception_type=None):
try:
result = self.example_python_code.expect_exceptions().run(
self.path.state, self.python)
result.exception_was_raised(exception_type, message.strip())
except ExpectedExceptionMessageWasDifferent as error:
if self.settings.get("rewrite"):
self.current_step.update(message=error.actual_message)
else:
raise
def file_contains(self, filename, contents):
assert (self.path.working_dir.joinpath(filename).bytes().decode("utf8")
== contents)
@validate(duration=Float())
def sleep(self, duration):
import time
time.sleep(duration)
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_failure(self, reason):
pass
def tear_down(self):
for vagrantfile in pathquery(self.path.state).named("Vagrantfile"):
Command("vagrant", "destroy",
"-f").in_dir(vagrantfile.abspath().dirname()).run()
