def load_scheme(yaml_text):
'''
Some optional keys have enforced default values, otherwise use dict.get()
'''
schema = MapPattern(
Str(),
Map({
'description':
Str(),
Optional('alias'):
Str(),
'version':
Str(),
'directives':
MapPattern(
Str(),
Map({
Optional('description'):
Str(),
'formats':
Seq(Enum(FORMATS)),
Optional('prepare'):
Map({
Optional('remove_whitespace', default=False): Bool(),
Optional('remove_characters', default=['']):
Seq(Str()),
Optional('strip_characters', default=['']): Seq(Str()),
}),
Optional('validate'):
Map({
Optional('alphabet'): Enum(ALPHABETS),
Optional('min_length'): Int(),
Optional('max_length'): Int(),
}),
Optional('target'):
Str(),
Optional('helper', default=False):
Bool()
}),
),
'algorithm':
Enum(ALGORITHMS),
'encodings':
MapPattern(
Str(),
Map({
'type':
Enum(ENCODINGS),
Optional('length', default=0):
Int(),
Optional('prefix', default=''):
Str(),
Optional('separator'):
Map({
'character': Str(),
'interval': Int()
})
}))
}))
return load(yaml_text, schema)
def _get_mapping(field: Fields, values: List[Values]) -> Map:
base_value_enum = Enum([str(el) for el in values])
if field == Fields.DATA_TYPES:
value_enum = base_value_enum | Seq(base_value_enum)
elif field == Fields.NUMBER_OF_COLUMNS:
value_enum = Int() | Seq(Int())
else:
value_enum = base_value_enum
conditions = Enum([str(el) for el in field.conditions()])
return Map({
"field": Enum(str(field)),
"condition": conditions,
"value": value_enum
})
def __init__(self, filename):
"""Load config from YAML file."""
filename = path.abspath(filename)
if filename is None:
self._config = []
else:
try:
with open(filename, 'r') as handle:
self._yaml = handle.read()
self._config = load(
self._yaml,
Seq(
Map({
Optional("name"):
"name",
"request":
Map({
Optional("path"):
Str(),
Optional("method"):
Enum([
"get",
"post",
"put",
"delete",
"GET",
"POST",
"PUT",
"DELETE",
]),
Optional("headers"):
MapPattern(Str(), Str()),
Optional("data"):
Str(),
}),
"response":
Map({
"content": Str() | Map({"file": Str()}),
Optional("code"): Int(),
Optional("headers"): MapPattern(Str(), Str()),
}),
})))
except Exception as e:
sys.stderr.write(
"Error reading YAML config file: {0}\n".format(str(e)))
sys.exit(1)
# Read and store all references to external content files
for pair in self._config:
content = pair.get('response', {}).get('content')
if type(content) != str and "file" in content:
with open(
path.join(path.dirname(filename), content['file']),
'r') as content_file_handle:
pair['response']['content'] = \
content_file_handle.read()
def load(path: Path, schema_pointer):
"""Load and validate .yaml file."""
schema = copy.deepcopy(schema_pointer)
with path.open() as f:
yaml = f.read()
data = yaml_load(yaml, Any())
is_template = path.name == "template.yaml"
# Replace real Country and Timezone values with fakes
if is_template:
schema["woo/woocommerce_default_country"] = Enum(["LL"])
schema["wp/timezone_string"] = Enum(["Region/Country"])
schema["wp/DEFAULT_WPLANG"] = Enum(["ll_LL"])
schema["woo/woocommerce_currency"] = Enum(["LLL"])
if "woo/woocommerce_tax_classes" in data:
# Inspect that tax classes and taxes match
# create enum for taxes from defined tax_classes
tax_classes = [
str(tax).lower().replace(" ", "-")
for tax in data["woo/woocommerce_tax_classes"]
]
# +1 is for standard schema which is never defined in tax class
for x in range(len(tax_classes) + 1):
# start counting with 1
schema[f"wootax/{x+1}"] = Map({
"country":
Enum(["LL"]) if is_template else Enum(COUNTRIES),
"state":
Str(),
"rate":
Decimal(),
"name":
Str(),
"priority":
Int(),
"compound":
Int(),
"shipping":
Int(),
"order":
Int(),
"class":
Enum([""]) if x == 0 else Enum(tax_classes),
"locations":
Map({}),
})
try:
return yaml_load(yaml, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
def get_type_schema_yaml_validator() -> Map:
seq_validator = Seq(
Map({
"field": Enum([str(el) for el in Fields]),
"condition": Str(),
"value": Str() | Seq(Str()),
}))
return Map({
Optional(str(RequirementTypes.INPUT_REQUIREMENTS)):
seq_validator,
Optional(str(RequirementTypes.OUTPUT_REQUIREMENTS)):
seq_validator,
})
class Engine(hitchpylibrarytoolkit.Engine):
info_definition = InfoDefinition(environments=InfoProperty(
Seq(Enum(["gui", "mac", "docker", "headless", "wsl"]))), )
def set_up(self):
self._build.ensure_built()
for filename, contents in self.given.get('files', {}).items():
filepath = self._build.working.parent.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(contents)
def screenshot_exists(self, filename):
assert self._build.working.joinpath(filename).exists()
def is_pipelines_config_valid(strictyaml_pipelines: YAML) -> YAML:
"""
TODO: Refactor to test and analyzer specific config validation.
"""
pipelines_schema = Map({
"pipelines":
Seq(
Map({
"name":
Str(),
"type":
Enum(["test", "analyzer"]),
Optional("coverage"):
Str(),
Optional("commands"):
Map({
"partial-scope": Str(),
"full-scope": Str()
}),
Optional("dirs"):
Seq(
Map({
"path": Str(),
Optional("full-scope", default=False): Bool()
})),
Optional("files"):
Seq(
Map({
"path": Str(),
Optional("full-scope", default=False): Bool()
}))
}))
})
try:
strictyaml_pipelines.revalidate(pipelines_schema)
return True
except YAMLValidationError:
return False
#tornado_server = tornado.httpserver.HTTPServer(app)
#tornado_server.bind(server.port)
#tornado_server.start(2)
#sys.stdout.write(message)
#sys.stdout.write("\n")
#sys.stdout.flush()
#tornado.ioloop.IOLoop.instance().start(1)
SCHEMA = Seq(
Map({
"request": Map({
"path": Str(),
"method": Enum(["get", "post", "put", "delete", "options"]),
}),
"response": Map({
"code": Int(),
"content": Str(),
}),
})
)
class Response(object):
def __init__(self, code, content):
self._code = code
self._content = content
@property
REPO_KEYS = {
"name": Str(), # Downloader doesn't use this, but cogs.red might.
"short": Str(),
"description": Str(),
"install_msg": Str(),
"author": Seq(Str()),
}
#: Metadata keys common to `shared_fields` and `cogs` schemas.
COMMON_KEYS = {
Optional("min_bot_version"): RedVersion(),
Optional("max_bot_version"): RedVersion(),
Optional("min_python_version"): PythonVersion(),
Optional("hidden", False): Bool(),
Optional("disabled", False): Bool(),
Optional("type", "COG"): Enum(["COG", "SHARED_LIBRARY"]),
}
#: `shared_fields` metadata keys.
SHARED_FIELDS_KEYS = {
"install_msg": Str(),
"author": Seq(Str()),
**COMMON_KEYS,
}
#: `cogs` metadata keys.
COG_KEYS = {
"name": Str(), # Downloader doesn't use this but I can set friendlier name
"short": Str(),
"description": Str(),
"end_user_data_statement": Str(),
class Experiment:
"""
This class orchestrates the analysis pipeline for our redox imaging experiments.
"""
experiment_schema = Map({
"pipeline":
Map({
"strategy": Str(),
"acquisition_method": Enum(["acquire", "mda"]),
"trimmed_profile_length": Int(),
"untrimmed_profile_length": Int(),
"seg_threshold": Int(),
"measurement_order": Int(),
"measure_thickness": Float(),
"reference_wavelength": Str(),
"image_register": Int(),
"channel_register": Int(),
"population_register": Int(),
"trimmed_regions": MapPattern(Str(), CommaSeparated(Float())),
"untrimmed_regions": MapPattern(Str(), CommaSeparated(Float())),
}),
"redox":
Map({
"ratio_numerator": Str(),
"ratio_denominator": Str(),
"r_min": Float(),
"r_max": Float(),
"instrument_factor": Float(),
"midpoint_potential": Float(),
"z": Int(),
"temperature": Float(),
}),
"registration":
Map({
"n_deriv": Float(),
"warp_n_basis": Float(),
"warp_order": Float(),
"warp_lambda": Float(),
"smooth_lambda": Float(),
"smooth_n_breaks": Float(),
"smooth_order": Float(),
"rough_lambda": Float(),
"rough_n_breaks": Float(),
"rough_order": Float(),
}),
"output":
Map({
"should_save_plots": Bool(),
"should_save_profile_data": Bool(),
"should_save_summary_data": Bool(),
}),
})
seg_images: xr.DataArray = None
rot_fl: xr.DataArray = None
rot_seg: xr.DataArray = None
midlines: xr.DataArray = None
untrimmed_raw_profiles: xr.DataArray = None
untrimmed_std_profiles: xr.DataArray = None
untrimmed_reg_profiles: xr.DataArray = None
trimmed_raw_profiles: xr.DataArray = None
trimmed_std_profiles: xr.DataArray = None
trimmed_reg_profiles: xr.DataArray = None
channel_warps: xr.DataArray = None
std_warps: xr.DataArray = None
def __init__(self, exp_dir):
self.experiment_dir = Path(exp_dir)
self.settings_path = self.experiment_dir.joinpath("settings.yaml")
try:
with open(self.settings_path, "r") as f:
self.config = load(f.read(), self.experiment_schema).data
except YAMLError:
raise ValueError("Incorrectly specified config file.")
self.experiment_id = self.experiment_dir.stem
# compute the filenames/paths for this experiment
self.movement_path = self.experiment_dir.joinpath(self.experiment_id +
"-mvmt.csv")
self.frame_map_path = self.experiment_dir.joinpath(self.experiment_id +
"-frame_map.csv")
self.processed_images_dir = self.experiment_dir.joinpath(
"processed_images")
self.rot_seg_dir = self.processed_images_dir.joinpath("rot_seg")
self.rot_fl_dir = self.processed_images_dir.joinpath("rot_fl")
self.fl_imgs_dir = self.processed_images_dir.joinpath(
"fluorescent_images")
self.orig_images_path = self.processed_images_dir.joinpath("images.nc")
self.seg_images_path = self.processed_images_dir.joinpath(
"seg_images.nc")
self.aligned_images_path = self.processed_images_dir.joinpath(
"aligned_images.nc")
self.aligned_seg_images_path = self.processed_images_dir.joinpath(
"aligned_seg_images.nc")
# load images
self.images = self._load_raw_images()
# try to load masks
try:
self.load_masks()
except IOError:
logging.info("No masks found in experiment directory")
pass
# Computed Filepaths
@property
def midlines_path(self) -> Path:
return self.analysis_dir.joinpath("midlines.pickle")
@property
def raw_img_stack_path(self) -> Path:
# TODO test that this works
accepted_extensions = [".tif", ".tiff", ".stk"]
candidate_paths = [
self.experiment_dir.joinpath(f"{self.experiment_id}{ext}")
for ext in accepted_extensions
]
for path in candidate_paths:
if path.exists():
return path
raise ValueError(
f"No image found in experiment directory. Tried the following files: {candidate_paths}"
)
@property
def fig_dir(self):
return self.analysis_dir.joinpath("figs")
def untrimmed_profile_data_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-untrimmed_{treatment}_profile_data.nc")
def trimmed_profile_data_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-trimmed_{treatment}_profile_data.nc")
@property
def channel_warp_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-channel_warps.nc")
@property
def std_warp_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id + "-std_warps.nc")
def untrimmed_profile_data_csv_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-untrimmed_{treatment}_profile_data.csv")
def trimmed_profile_data_csv_path(self, treatment="raw"):
return self.analysis_dir.joinpath(
self.experiment_id + f"-trimmed_{treatment}_profile_data.csv")
@property
def untrimmed_region_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-untrimmed_region_data.csv")
@property
def trimmed_region_data_path(self):
return self.analysis_dir.joinpath(self.experiment_id +
"-trimmed_region_data.csv")
@property
def analysis_dir(self) -> Path:
date_str = datetime.datetime.now().strftime("%Y-%m-%d")
strategy = self.config["pipeline"]["strategy"]
if len(strategy) > 0:
suffix = f"_{strategy}"
else:
suffix = ""
analysis_dir_ = self.experiment_dir.joinpath(
"analyses",
utils.get_valid_filename(f"{date_str}{suffix}"),
)
# analysis_dir_.mkdir(parents=True, exist_ok=True)
return analysis_dir_
def _load_raw_images(self):
"""
This returns the raw (non-median-subtracted) images
"""
logging.info(f"Loading image data from {self.raw_img_stack_path}")
raw_image_data = pio.load_tiff_as_hyperstack(
img_stack_path=self.raw_img_stack_path,
manual_metadata=self.frame_map_path,
mvmt_metadata=self.movement_path,
)
raw_image_data = raw_image_data.assign_coords({
"experiment_id": (
("animal", ),
np.repeat(self.experiment_id, raw_image_data.animal.size),
)
})
raw_image_data = self.add_experiment_metadata_to_data_array(
raw_image_data)
return raw_image_data
def _load_movement(self) -> pd.DataFrame:
movement_path = self.experiment_dir.joinpath(self.experiment_id +
"-mvmt.csv")
try:
df = pd.read_csv(movement_path)
df = df.pivot_table(index="animal",
columns=["region", "pair"],
values="movement")
df = df.stack("pair")
return df
except FileNotFoundError:
logging.warning(
f"Tried to access {movement_path}; file was not found")
return None
def make_analysis_dir(self) -> None:
logging.info(f"Making analysis directory at {self.analysis_dir}")
self.analysis_dir.mkdir(parents=True, exist_ok=True)
@property
def trimmed_summary_table(self):
df = profile_processing.summarize_over_regions(
self.trimmed_raw_profiles,
regions=self.config["pipeline"]["trimmed_regions"],
rescale=False,
**self.config["redox"],
)
return df
@property
def untrimmed_summary_table(self):
df = profile_processing.summarize_over_regions(
self.untrimmed_raw_profiles,
regions=self.config["pipeline"]["untrimmed_regions"],
**self.config["redox"],
)
return df
####################################################################################
# PIPELINE
####################################################################################
def full_pipeline(self):
logging.info(f"Starting full pipeline run for {self.experiment_dir}")
self.make_analysis_dir()
logging.info(f"Saving fluorescent images to {self.fl_imgs_dir}")
pio.save_images_xarray_to_tiffs(self.images,
self.fl_imgs_dir,
prefix=self.experiment_id)
self.segment_pharynxes()
self.register_images()
self.align_and_center()
self.calculate_midlines()
self.measure_under_midlines()
self.register_profiles()
self.trim_data()
self.calculate_redox()
self.do_manual_ap_flips()
self.persist_to_disk()
logging.info(f"Finished full pipeline run for {self.experiment_dir}")
return self
def run_neuron_pipeline(self):
logging.info(
f"Starting full neuron analysis pipeline run for {self.experiment_dir}"
)
self.make_analysis_dir()
df = ip.measure_under_labels(self.images,
self.seg_images).reset_index()
df.to_csv(self.analysis_dir /
(self.experiment_id + "-neuron_analysis.csv"))
def segment_pharynxes(self):
if self.seg_images is not None:
logging.info("masks have been specified. skipping mask generation")
self.save_masks()
return
else:
logging.info("Generating masks")
self.seg_images = ip.segment_pharynxes(
self.images,
wvl=self.config["pipeline"]["reference_wavelength"])
self.save_masks()
def register_images(self):
if self.config["pipeline"]["image_register"]:
logging.info("Registering Images")
self.images = ip.register_all_images(self.images, self.seg_images)
def align_and_center(self):
logging.info("Centering and rotating pharynxes")
self.rot_fl, self.rot_seg = ip.center_and_rotate_pharynxes(
self.images,
self.seg_images,
)
logging.info(f"Saving rotated FL images to {self.aligned_images_path}")
pio.save_profile_data(self.rot_fl, self.aligned_images_path)
logging.info(f"Saving rotated masks to {self.aligned_seg_images_path}")
pio.save_profile_data(self.rot_seg, self.aligned_seg_images_path)
def calculate_midlines(self):
logging.info("Calculating midlines")
self.midlines = ip.calculate_midlines(self.rot_seg, degree=4)
def measure_under_midlines(self):
logging.info("Measuring under midlines")
self.untrimmed_raw_profiles = ip.measure_under_midlines(
self.rot_fl,
self.midlines,
n_points=self.config["pipeline"]["untrimmed_profile_length"],
order=self.config["pipeline"]["measurement_order"],
thickness=float(self.config["pipeline"]["measure_thickness"]),
)
self.untrimmed_raw_profiles = profile_processing.align_pa(
self.untrimmed_raw_profiles)
self.untrimmed_raw_profiles = self.add_experiment_metadata_to_data_array(
self.untrimmed_raw_profiles)
# subtract the image medians from the profile data
logging.info("Subtracting image medians from profile data")
self.untrimmed_raw_profiles = ip.subtract_medians(
self.untrimmed_raw_profiles, self.images)
def register_profiles(self):
if self.config["pipeline"]["population_register"]:
logging.info("Standardizing profiles")
(
self.untrimmed_std_profiles,
self.std_warps,
) = profile_processing.standardize_profiles(
self.untrimmed_raw_profiles,
redox_params=self.config["redox"],
**self.config["registration"],
)
if self.config["pipeline"]["channel_register"]:
logging.info("Channel-Registering profiles")
if self.untrimmed_std_profiles is not None:
logging.info(
"using the standardize profiles for channel-registration")
data_to_register = self.untrimmed_std_profiles
else:
logging.info("using the raw profiles for channel-registration")
data_to_register = self.untrimmed_raw_profiles
(
self.untrimmed_reg_profiles,
self.channel_warps,
) = profile_processing.channel_register(
data_to_register,
redox_params=self.config["redox"],
reg_params=self.config["registration"],
)
def trim_data(self):
logging.info("Trimming intensity data")
self.trimmed_raw_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_raw_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
if self.untrimmed_std_profiles is not None:
self.trimmed_std_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_std_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
if self.untrimmed_reg_profiles is not None:
self.trimmed_reg_profiles = self.add_experiment_metadata_to_data_array(
profile_processing.trim_profiles(
self.untrimmed_reg_profiles,
self.config["pipeline"]["seg_threshold"],
ref_wvl=self.config["pipeline"]["reference_wavelength"],
))
def calculate_redox(self):
logging.info("Calculating redox measurements")
redox_params = self.config["redox"]
# Images
self.images = utils.add_derived_wavelengths(self.images,
**redox_params)
self.rot_fl = utils.add_derived_wavelengths(self.rot_fl,
**redox_params)
# profiles
self.trimmed_raw_profiles = utils.add_derived_wavelengths(
self.trimmed_raw_profiles, **redox_params)
self.untrimmed_raw_profiles = utils.add_derived_wavelengths(
self.untrimmed_raw_profiles, **redox_params)
def do_manual_ap_flips(self):
# TODO finish implementation
logging.info("skipping manual AP flips - not implemented")
def flip_at(self, idx):
# TODO finish implementation
raise NotImplementedError
####################################################################################
# PERSISTENCE / IO
####################################################################################
def save_images(self):
"""Save this experiment's images to disk as netCDF4 files"""
imgs_paths = [
(self.images, self.orig_images_path),
(self.rot_fl, self.aligned_images_path),
(self.seg_images, self.seg_images_path),
(self.rot_seg, self.aligned_seg_images_path),
]
for img, path in imgs_paths:
if img is not None:
logging.info(f"Saving images to {path}")
img.to_netcdf(path)
# def load_tiff_as_hyperstack(self):
# pass
def make_fig_dir(self):
fig_dir = self.analysis_dir.joinpath("figs")
fig_dir.mkdir(parents=True, exist_ok=True)
return fig_dir
def save_individual_profiles(self, profile_data, treatment: str,
trimmed: bool):
if profile_data is None:
return
fig_dir = self.make_fig_dir()
profile_data_fig_dir = (fig_dir / "profile_data" / treatment /
("trimmed" if trimmed else "untrimmed"))
individual_data_fig_dir = profile_data_fig_dir.joinpath("inividual")
individual_data_fig_dir.mkdir(exist_ok=True, parents=True)
for title, fig in plots.generate_wvl_pair_timepoint_profile_plots(
profile_data):
title = title.replace(" ", "")
fig.savefig(individual_data_fig_dir /
f"{self.experiment_id}-{title}-individuals.pdf")
plt.close(fig)
def save_avg_profiles(self, profile_data, treatment: str, trimmed: bool):
if profile_data is None:
return
fig_dir = self.make_fig_dir()
profile_data_fig_dir = (fig_dir / "profile_data" / treatment /
("trimmed" if trimmed else "untrimmed"))
individual_data_fig_dir = profile_data_fig_dir.joinpath("avg")
individual_data_fig_dir.mkdir(exist_ok=True, parents=True)
for title, fig in plots.generate_avg_wvl_pair_profile_plots(
profile_data):
title = title.replace(" ", "")
fig.savefig(individual_data_fig_dir /
f"{self.experiment_id}-{title}-avg.pdf")
plt.close(fig)
def save_plots(self):
with warnings.catch_warnings():
warnings.simplefilter("ignore")
for data, treatment, trimmed in [
(self.untrimmed_raw_profiles, "raw", False),
(self.untrimmed_std_profiles, "standardized", False),
(self.untrimmed_reg_profiles, "channel-registered", False),
(self.trimmed_raw_profiles, "raw", True),
(self.trimmed_std_profiles, "standardized", True),
(self.trimmed_reg_profiles, "channel-registered", True),
]:
self.save_individual_profiles(data, treatment, trimmed)
self.save_avg_profiles(data, treatment, trimmed)
# frame-normed Ratio Images
mvmt_annotation_img_path = self.fig_dir.joinpath(
f"{self.experiment_id}-movement_annotation_imgs.pdf")
imgs = utils.add_derived_wavelengths(self.images,
**self.config["redox"])
with PdfPages(mvmt_annotation_img_path) as pdf:
for i in tqdm(range(self.images.animal.size)):
fig = plots.plot_pharynx_R_imgs(imgs[i],
mask=self.seg_images[i])
fig.suptitle(f"animal = {i}")
pdf.savefig(fig)
if (i % 20) == 0:
plt.close("all")
# Pop-normed ratio images
u = self.trimmed_raw_profiles.sel(wavelength="r").mean()
std = self.trimmed_raw_profiles.sel(wavelength="r").std()
for pair in self.rot_fl.pair.values:
for tp in self.rot_fl.timepoint.values:
ratio_img_path = self.fig_dir.joinpath(
f"{self.experiment_id}-ratio_images-pair={pair};timepoint={tp}.pdf"
)
with PdfPages(ratio_img_path) as pdf:
logging.info(
f"Saving ratio images to {ratio_img_path}")
for i in tqdm(range(self.rot_fl.animal.size)):
fig, ax = plt.subplots(dpi=300)
ratio_img = (self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_numerator"],
pair=pair,
timepoint=tp,
) / self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_denominator"],
pair=pair,
timepoint=tp,
))[i]
fl_img = self.rot_fl.sel(
wavelength=self.config["redox"]
["ratio_numerator"],
pair=pair,
timepoint=tp,
)[i]
im, cbar = plots.imshow_ratio_normed(
ratio_img,
fl_img,
r_min=u - (std * 1.96),
r_max=u + (std * 1.96),
colorbar=True,
i_max=5000,
i_min=1000,
ax=ax,
)
ax.plot(
*self.midlines.sel(
pair=pair,
timepoint=tp,
)[i].values[()].linspace(),
color="green",
alpha=0.3,
)
strain = self.rot_fl.strain.values[i]
ax.set_title(
f"Animal={i} ; Pair={pair} ; Strain={strain}")
cax = cbar.ax
for j in range(len(self.trimmed_raw_profiles)):
cax.axhline(
self.trimmed_raw_profiles.sel(
wavelength="r",
pair=pair,
timepoint=tp)[j].mean(),
color="k",
alpha=0.1,
)
cax.axhline(
self.trimmed_raw_profiles.sel(
wavelength="r", pair=pair,
timepoint=tp)[i].mean(),
color="k",
)
pdf.savefig()
if (i % 20) == 0:
plt.close("all")
def persist_profile_data(self):
for treatment, untrimmed_profile_data in (
("raw", self.untrimmed_raw_profiles),
("std", self.untrimmed_std_profiles),
("reg", self.untrimmed_reg_profiles),
):
if untrimmed_profile_data is not None:
untrimmed_prof_path = self.untrimmed_profile_data_path(
treatment)
logging.info(
f"Saving untrimmed {treatment} profile data to {untrimmed_prof_path}"
)
pio.save_profile_data(untrimmed_profile_data,
untrimmed_prof_path)
untrimmed_prof_path_csv = self.untrimmed_profile_data_csv_path(
treatment)
profile_processing.to_dataframe(
untrimmed_profile_data,
"value").to_csv(untrimmed_prof_path_csv)
for treatment, trimmed_profile_data in (
("raw", self.trimmed_raw_profiles),
("std", self.trimmed_std_profiles),
("reg", self.trimmed_reg_profiles),
):
if trimmed_profile_data is not None:
trimmed_prof_path = self.trimmed_profile_data_path(treatment)
logging.info(
f"Saving trimmed {treatment} profile data to {trimmed_prof_path}"
)
pio.save_profile_data(trimmed_profile_data, trimmed_prof_path)
trimmed_prof_path_csv = self.trimmed_profile_data_csv_path(
treatment)
logging.info(
f"Saving trimmed {treatment} profile data to {trimmed_prof_path_csv}"
)
profile_processing.to_dataframe(
trimmed_profile_data,
"value").to_csv(trimmed_prof_path_csv)
# Warps, if necessary
if self.config["pipeline"]["channel_register"]:
logging.info(
f"Saving channel warp data to {self.channel_warp_data_path}")
self.channel_warps.to_netcdf(self.channel_warp_data_path)
if self.config["pipeline"]["population_register"]:
logging.info(
f"Saving channel warp data to {self.std_warp_data_path}")
self.std_warps.to_netcdf(self.std_warp_data_path)
def save_summary_data(self):
# Persist the region means
logging.info(
f"Saving untrimmed region means to {self.untrimmed_region_data_path}"
)
self.untrimmed_summary_table.to_csv(self.untrimmed_region_data_path)
logging.info(
f"Saving trimmed region means to {self.trimmed_region_data_path}")
self.trimmed_summary_table.to_csv(self.trimmed_region_data_path)
def save_masks(self):
logging.info(f"saving masks to {self.seg_images_path}")
pio.save_profile_data(self.seg_images, self.seg_images_path)
def load_masks(self):
self.seg_images = pio.load_profile_data(self.seg_images_path)
logging.info(f"Loaded masks from {self.seg_images_path}")
def save_midlines(self):
pio.save_midlines(self.midlines_path, self.midlines)
def load_midlines(self):
return pio.load_midlines(self.midlines_path)
def persist_to_disk(self):
logging.info(
f"Saving {self.experiment_id} inside {self.experiment_dir}")
self.save_midlines()
if self.config["output"]["should_save_summary_data"]:
self.save_summary_data()
if self.config["output"]["should_save_profile_data"]:
self.persist_profile_data()
if self.config["output"]["should_save_plots"]:
self.save_plots()
####################################################################################
# MISC / HELPER
####################################################################################
def add_experiment_metadata_to_data_array(self, data_array: xr.DataArray):
params = {}
params.update(self.config["pipeline"])
params.update(self.config["redox"])
params.update(self.config["registration"])
to_remove = ["trimmed_regions", "untrimmed_regions"]
for k in to_remove:
del params[k]
return data_array.assign_attrs(**params)
limits_schema = Map(
{
"time": Int(),
Optional("memory", default=8192): Int(),
Optional("output"): Int(),
Optional("cores"): Int(),
}
)
module_schema = Regex(r"\.?\w+(\.\w+)*")
plugin_schema = Map(
{"module": module_schema, Optional("config"): MapPattern(Str(), Any())}
)
task_sources = Enum(["local", "url"])
schema = Map(
{
"title": Str(),
Optional("description"): Str(),
"limits": limits_schema,
"steps": Map(
{"run": Seq(plugin_schema), Optional("analysis"): Seq(plugin_schema)}
),
"observers": Seq(plugin_schema),
"tasks": MapPattern(Str(), MapPattern(Str(), Any())),
"tools": MapPattern(
Str(),
Map(
{
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
yaml_snippet=GivenProperty(
Str(), document="yaml_snippet:\n```yaml\n{{ yaml_snippet }}\n```"),
yaml_snippet_1=GivenProperty(
Str(),
document="yaml_snippet_1:\n```yaml\n{{ yaml_snippet_1 }}\n```"),
yaml_snippet_2=GivenProperty(
Str(),
document="yaml_snippet_2:\n```yaml\n{{ yaml_snippet_2 }}\n```"),
modified_yaml_snippet=GivenProperty(
Str(),
document=
"modified_yaml_snippet:\n```yaml\n{{ modified_yaml_snippet }}\n```"
),
python_version=GivenProperty(Str()),
ruamel_version=GivenProperty(Str()),
setup=GivenProperty(Str(), document="```python\n{{ setup }}\n```"),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
fails_on_python_2=InfoProperty(schema=Bool()),
description=InfoProperty(schema=Str()),
experimental=InfoProperty(schema=Bool()),
)
def __init__(self,
keypath,
python_path=None,
rewrite=False,
cprofile=False):
self.path = keypath
self._python_path = python_path
self._rewrite = rewrite
self._cprofile = cprofile
def set_up(self):
"""Set up your applications and the test environment."""
self.path.profile = self.path.gen.joinpath("profile")
if not self.path.profile.exists():
self.path.profile.mkdir()
if not self._python_path:
self.python = hitchpylibrarytoolkit.project_build(
"strictyaml",
self.path,
self.given["python version"],
{
"ruamel.yaml": self.given["ruamel version"]
},
).bin.python
else:
self.python = Path(self._python_path)
assert self.python.exists()
self.example_py_code = (ExamplePythonCode(
self.python, self.path.gen).with_code(self.given.get(
"code", "")).with_setup_code(self.given.get(
"setup", "")).with_terminal_size(160, 100).with_strings(
yaml_snippet_1=self.given.get("yaml_snippet_1"),
yaml_snippet=self.given.get("yaml_snippet"),
yaml_snippet_2=self.given.get("yaml_snippet_2"),
modified_yaml_snippet=self.given.get(
"modified_yaml_snippet"),
))
@no_stacktrace_for(AssertionError)
@no_stacktrace_for(HitchRunPyException)
@validate(
code=Str(),
will_output=Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
raises=Map({
Optional("type"): CODE_TYPE,
Optional("message"): CODE_TYPE
}),
in_interpreter=Bool(),
)
def run(
self,
code,
will_output=None,
yaml_output=True,
raises=None,
in_interpreter=False,
):
if in_interpreter:
code = "{0}\nprint(repr({1}))".format(
"\n".join(code.strip().split("\n")[:-1]),
code.strip().split("\n")[-1])
to_run = self.example_py_code.with_code(code)
if self._cprofile:
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug)))
if raises is None:
result = (to_run.expect_exceptions().run()
if raises is not None else to_run.run())
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")])
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(
**{"will output": actual_output})
else:
raise
elif raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2") else
exception_type["in python 3"])
if message is not None:
if not isinstance(message, str):
differential = True
message = (message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"])
try:
result = to_run.expect_exceptions().run()
result.exception_was_raised(exception_type, message)
except ExpectedExceptionMessageWasDifferent as error:
if self._rewrite and not differential:
new_raises = raises.copy()
new_raises["message"] = result.exception.message
self.current_step.update(raises=new_raises)
else:
raise
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_success(self):
if self._rewrite:
self.new_story.save()
if self._cprofile:
self.python(
self.path.key.joinpath("printstats.py"),
self.path.profile.joinpath("{0}.dat".format(self.story.slug)),
).run()
from typing import Sequence
import strictyaml
from strictyaml import Any, Enum, Map, MapPattern, Seq, Str
from labby.hw.core import Device
SCHEMA = Map({
"devices":
Seq(
Map({
"name": Str(),
"type": Enum("power_supply"),
"driver": Str(),
"args": MapPattern(Str(), Any()),
})),
})
class Config:
config: strictyaml.YAML
devices: Sequence[Device]
def __init__(self, yaml_contents: str) -> None:
self.config = strictyaml.load(yaml_contents, SCHEMA)
self.devices = [
Device.create(device["name"], device["driver"].data,
device["args"].data)
for device in self.config["devices"]
]
Opt("mail"):
Map({
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
})
})
_job_defaults_common = {
Opt("shell"):
Str(),
Opt("concurrencyPolicy"):
Enum(['Allow', 'Forbid', 'Replace']),
Opt("captureStderr"):
Bool(),
Opt("captureStdout"):
Bool(),
Opt("saveLimit"):
Int(),
Opt("failsWhen"):
Map({
"producesStdout": Bool(),
Opt("producesStderr"): Bool(),
Opt("nonzeroReturn"): Bool(),
}),
Opt("onFailure"):
Map({
Opt("retry"):
from textwrap import dedent
import pytest
from lxml import html
from strictyaml import Enum, Map, Optional, Seq, Str, Url
from juniorguru.scrapers.pipelines import sections_parser
from juniorguru.scrapers.pipelines.sections_parser import (ListSection,
TextFragment)
from utils import (load_yaml, param_startswith_skip, param_xfail_missing,
startswith_skip)
schema = Seq(
Map({
Optional('heading'): Str(),
'type': Enum(['paragraph', 'list']),
'contents': Seq(Str()),
}))
def generate_params(fixtures_dirname):
for html_path in (Path(__file__).parent /
fixtures_dirname).rglob('*.html'):
if startswith_skip(html_path):
yield param_startswith_skip(path)
else:
yml_path = html_path.with_suffix('.yml')
if startswith_skip(yml_path):
yield param_startswith_skip(path)
elif yml_path.is_file():
yield pytest.param(html_path.read_text(),
Optional("year"): Str(),
Optional("month"): Str(),
Optional("day"): Str(),
Optional("week"): Str(),
Optional("day_of_week"): Str(),
Optional("hour"): Str(),
Optional("minute"): Str(),
Optional("second"): Str(),
}),
}),
}),
Optional("filter"):
Map({
Optional("taxo_exclude"): Seq(Str()),
Optional("territorial_unit_ids"): Seq(Str()),
Optional("json_format", default="short"): Enum(["short", "long"]),
Optional("start_date"): Datetime(),
Optional("end_date"): Datetime(),
Optional("type_date", default="sighting"): Enum(["sighting", "entry"]),
}),
"site":
MapPattern(
Str(),
Map({
"enabled": Bool(),
"site": Url(),
"user_email": Email(),
"user_pw": Str(),
"client_key": Str(),
"client_secret": Str(),
}),
class Engine(BaseEngine):
"""Python engine for running tests."""
schema = StorySchema(
preconditions=Map({
"example.yaml": Str(),
"code": Str(),
"analysis_code": Str(),
}),
params=Map({
"python version": Str(),
}),
about={
"description": Str(),
Optional("importance"): Int(),
},
)
def __init__(self, keypath, settings):
self.path = keypath
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree()
self.path.state.mkdir()
self.python_package = hitchpython.PythonPackage(
self.preconditions.get('python_version', '3.5.0'))
self.python_package.build()
self.pip = self.python_package.cmd.pip
self.python = self.python_package.cmd.python
# Install debugging packages
with hitchtest.monitor(
[self.path.key.joinpath("debugrequirements.txt")]) as changed:
if changed:
run(
self.pip("install", "-r",
"debugrequirements.txt").in_dir(self.path.key))
# Uninstall and reinstall
with hitchtest.monitor(
pathq(self.path.project.joinpath("hitchhttp")).ext(
"py")) as changed:
if changed:
run(self.pip("uninstall", "hitchhttp", "-y").ignore_errors())
run(self.pip("install", ".").in_dir(self.path.project))
self.path.state.joinpath("example.yaml").write_text(
self.preconditions['example.yaml'])
self.path.state.joinpath("myserver.py").write_text(
self.preconditions['code'])
self.services = hitchserve.ServiceBundle(
str(self.path.project),
startup_timeout=2.0,
shutdown_timeout=2.0,
)
def server_starts(self, message=""):
self.services['Server'] = hitchserve.Service(
command=self.python(
self.path.state.joinpath("myserver.py")).in_dir(
self.path.state),
log_line_ready_checker=lambda line: message in line,
)
self.services.startup(interactive=False)
@validate(request=Map({
"method": Enum(["GET", "POST"]),
"url": Str(),
}),
expected_response=Map({
"code": Int(),
"content": Str(),
}))
def request(self, request, expected_response=None):
import requests
response = requests.request(
request['method'].lower(),
request['url'],
#data=data,
#headers=headers,
#timeout=timeout
)
if expected_response is not None:
assert expected_response['code'] == response.status_code
try:
assert expected_response['content'] == response.content.decode(
'utf8')
except AssertionError:
raise AssertionError("{0} expected, saw {1}".format(
expected_response['content'],
response.content.decode('utf8')))
def server_stopped_normally(self):
if self.services is not None:
self.services.shutdown()
def run_command(self, command):
self.ipython_step_library.run(command)
self.doc.step("code", command=command)
def code(self, command):
self.ipython_step_library.run(command)
self.doc.step("code", command=command)
#@validate(exception=Str())
#def raises_exception(self, command, exception, why=''):
#"""
#Command raises exception.
#"""
#import re
#self.error = self.ipython_step_library.run(
#command, swallow_exception=True
#).error
#if self.error is None:
#raise Exception("Expected exception, but got none")
#full_exception = re.compile("(?:\\x1bs?\[0m)+(?:\n+)+{0}".format(
#re.escape("\x1b[0;31m"))
#).split(self.error)[-1]
#exception_class_name, exception_text = full_exception.split("\x1b[0m: ")
#if self.settings.get("overwrite"):
#self.current_step.update(exception=str(exception_text))
#else:
#assert exception.strip() in exception_text, "UNEXPECTED:\n{0}".format(exception_text)
#self.doc.step(
#"exception",
#command=command,
#exception_class_name=exception_class_name,
#exception=exception_text,
#why=why,
#)
def raises_exception(self, exception):
"""
Expect an exception.
"""
class ExpectedExceptionDidNotHappen(Exception):
pass
error_path = self.path.state.joinpath("error.txt")
runpy = self.path.state.joinpath("runmypy.py")
if error_path.exists():
error_path.remove()
env = Environment()
env.loader = DictLoader(
load(
self.path.key.joinpath("codetemplates.yml").bytes().decode(
'utf8')).data)
runpy.write_text(
env.get_template("raises_exception").render(
setup=self.preconditions['setup'],
code=self.preconditions['code'],
variables=self.preconditions.get('variables', None),
yaml_snippet=self.preconditions.get("yaml_snippet"),
modified_yaml_snippet=self.preconditions.get(
"modified_yaml_snippet"),
exception=exception,
error_path=error_path,
))
self.python(runpy).run()
if not error_path.exists():
raise ExpectedExceptionDidNotHappen()
else:
assert exception.strip() in error_path.bytes().decode(
'utf8'), "expected:\n{0}\nshould be:\n{1}".format(
exception,
error_path.bytes().decode('utf8'),
)
def should_be_equal_to(self, rhs):
"""
Code should be equal to rhs
"""
class UnexpectedException(Exception):
pass
error_path = self.path.state.joinpath("error.txt")
runpy = self.path.state.joinpath("runmypy.py")
if error_path.exists():
error_path.remove()
env = Environment()
env.loader = DictLoader(
load(
self.path.key.joinpath("codetemplates.yml").bytes().decode(
'utf8')).data)
runpy.write_text(
env.get_template("shouldbeequal").render(
setup=self.preconditions['setup'],
code=self.preconditions['code'],
variables=self.preconditions.get('variables', None),
yaml_snippet=self.preconditions.get("yaml_snippet"),
modified_yaml_snippet=self.preconditions.get(
"modified_yaml_snippet"),
rhs=rhs,
error_path=error_path,
))
self.python(runpy).run()
if error_path.exists():
raise UnexpectedException(error_path.bytes().decode("utf8"))
def run_analysis_code(self):
self.path.state.joinpath("analysis.py").write_text(
self.preconditions['analysis_code'])
self.python(self.path.state.joinpath("analysis.py")).run()
def returns_true(self, command, why=''):
self.ipython_step_library.assert_true(command)
self.doc.step("true", command=command, why=why)
def should_be_equal(self, lhs='', rhs='', why=''):
command = """({0}).should.be.equal({1})""".format(lhs, rhs)
self.ipython_step_library.run(command)
self.doc.step("true", command=command, why=why)
def assert_true(self, command):
self.ipython_step_library.assert_true(command)
self.doc.step("true", command=command)
def assert_exception(self, command, exception):
error = self.ipython_step_library.run(command,
swallow_exception=True).error
assert exception.strip() in error
self.doc.step("exception", command=command, exception=exception)
def on_failure(self, result):
if self.settings.get("pause_on_failure", True):
if self.preconditions.get("launch_shell", False):
self.services.log(message=self.stacktrace.to_template())
self.shell()
def shell(self):
if hasattr(self, 'services'):
self.services.start_interactive_mode()
import sys
import time
time.sleep(0.5)
if path.exists(
path.join(path.expanduser("~"),
".ipython/profile_default/security/",
self.ipython_kernel_filename)):
call([
sys.executable, "-m", "IPython", "console", "--existing",
"--no-confirm-exit",
path.join(path.expanduser("~"),
".ipython/profile_default/security/",
self.ipython_kernel_filename)
])
else:
call([
sys.executable, "-m", "IPython", "console", "--existing",
self.ipython_kernel_filename
])
self.services.stop_interactive_mode()
def assert_file_contains(self, filename, contents):
assert self.path.state.joinpath(filename).bytes().decode(
'utf8').strip() == contents.strip()
self.doc.step("filename contains",
filename=filename,
contents=contents)
def pause(self, message="Pause"):
if hasattr(self, 'services'):
self.services.start_interactive_mode()
import IPython
IPython.embed()
if hasattr(self, 'services'):
self.services.stop_interactive_mode()
def on_success(self):
if self.settings.get("overwrite"):
self.new_story.save()
def tear_down(self):
try:
self.shutdown_connection()
except:
pass
if hasattr(self, 'services'):
self.services.shutdown()
class WooSchema:
"""Schema for localization YAML files."""
# https://github.com/woocart/woocart-defaults/blob/master/src/importers/class-woopage.php#L14
productMeta = {
"title": Str(),
"description": Str(),
Optional("price"): Str(),
Optional("category"): Str(),
"images": Seq(Str()),
}
# https://github.com/woocart/woocart-defaults/blob/master/src/importers/class-woopage.php#L14
pageMeta = {
"post_title": Str(),
Optional("post_name"): Str(),
Optional("post_excerpt"): Str(),
"post_status": Enum(["draft", "publish"]),
"post_type": Enum(["page", "post"]),
Optional("post_category"): Str(),
Optional("meta_input"): MapPattern(Str(), Str()),
Optional("woocart_defaults"): MapPattern(Str(), Str()),
}
localization = {
"woo/woocommerce_default_country":
Enum(COUNTRIES),
"wp/date_format":
Enum(["d/m/Y", "Y-m-d", "F j, Y", "m/d/Y"]),
"wp/time_format":
Enum(["H:i", "g:i A"]),
"wp/start_of_week":
Enum(["1", "2", "3", "4", "5", "6", "7"]),
"wp/timezone_string":
Enum(TIMEZONES),
"wp/blog_charset":
Enum(["UTF-8"]),
"wp/DEFAULT_WPLANG":
Enum(WPLANGS),
Optional("wp/blogdescription"):
Str(),
Optional("wp/woocommerce_demo_store_notice"):
Str(),
"woo/woocommerce_weight_unit":
Enum(["kg", "k", "lbs", "oz"]),
"woo/woocommerce_dimension_unit":
Enum(["m", "cm", "mm", "in", "yd"]),
"woo/woocommerce_currency":
Enum(CURRENCIES),
"woo/woocommerce_currency_pos":
Enum(["right_space", "left_space", "left", "right"]),
"woo/woocommerce_price_thousand_sep":
Enum([".", ","]),
"woo/woocommerce_price_decimal_sep":
Enum([",", "."]),
"woo/woocommerce_price_num_decimals":
Enum(["2"]),
Optional("woo/woocommerce_tax_classes"):
Seq(Str()),
"woo/woocommerce_bacs_settings":
Map({
"enabled": Bool(),
Optional("title"): Str(),
Optional("description"): Str(),
Optional("instructions"): Str(),
Optional("account_name"): Str(),
Optional("account_number"): Str(),
Optional("sort_code"): Str(),
Optional("bank_name"): Str(),
Optional("iban"): Str(),
Optional("bic"): Str(),
Optional("account_details"): Str(),
}),
"woo/woocommerce_cod_settings":
Map({
"enabled": Bool(),
Optional("title"): Str(),
Optional("description"): Str(),
Optional("instructions"): Str(),
Optional("enable_for_methods"): Str(),
Optional("enable_for_virtual"): Bool(),
}),
"woo/woocommerce_checkout_privacy_policy_text":
Str(),
"woo/woocommerce_registration_privacy_policy_text":
Str(),
".woo/woocommerce_bacs_settings_format":
Enum(["serialized"]),
".woo/woocommerce_cod_settings_format":
Enum(["serialized"]),
Optional(".woo/woocommerce_tax_classes_format"):
Enum(["implode_newline"]),
}
@staticmethod
def load(path: Path, schema_pointer):
"""Load and validate .yaml file."""
schema = copy.deepcopy(schema_pointer)
with path.open() as f:
yaml = f.read()
data = yaml_load(yaml, Any())
is_template = path.name == "template.yaml"
# Replace real Country and Timezone values with fakes
if is_template:
schema["woo/woocommerce_default_country"] = Enum(["LL"])
schema["wp/timezone_string"] = Enum(["Region/Country"])
schema["wp/DEFAULT_WPLANG"] = Enum(["ll_LL"])
schema["woo/woocommerce_currency"] = Enum(["LLL"])
if "woo/woocommerce_tax_classes" in data:
# Inspect that tax classes and taxes match
# create enum for taxes from defined tax_classes
tax_classes = [
str(tax).lower().replace(" ", "-")
for tax in data["woo/woocommerce_tax_classes"]
]
# +1 is for standard schema which is never defined in tax class
for x in range(len(tax_classes) + 1):
# start counting with 1
schema[f"wootax/{x+1}"] = Map({
"country":
Enum(["LL"]) if is_template else Enum(COUNTRIES),
"state":
Str(),
"rate":
Decimal(),
"name":
Str(),
"priority":
Int(),
"compound":
Int(),
"shipping":
Int(),
"order":
Int(),
"class":
Enum([""]) if x == 0 else Enum(tax_classes),
"locations":
Map({}),
})
try:
return yaml_load(yaml, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
@staticmethod
def load_string(data: bytes, schema, path: str):
"""Load and validate yaml data."""
try:
return yaml_load(data, Map(schema), path)
except YAMLError:
raise
return as_document(schema)
environment_property_description_schema = Map({
"position":
Map({
"value": Float(),
"unit": Str()
}),
"temperature":
Map({
"value": Float(),
"unit": Str()
}),
"type":
Enum([
'walls_and_environment_heat_transfer_coefficient',
'walls_and_water_heat_transfer_coefficient_model',
'walls_and_air_heat_transfer_coefficient_model',
'overall_heat_transfer_coefficient_model',
'walls_without_environment_heat_transfer_coefficient'
]),
Optional("heat_transfer_coefficient"):
Map({
"value": Float(),
"unit": Str()
}),
Optional("overall_heat_transfer_coefficient"):
Map({
"value": Float(),
"unit": Str()
}),
Optional("fluid_velocity"):
Map({
Str(
doc={
"title":
"source",
"text":
"The name of the resource that sends messages or events to the stream processor"
}),
"window":
Map(
{
"type":
Enum(
WINDOW_TYPES,
doc={
"title":
"type",
"text":
"The type of temporal window; must be one of `sliding` or `stagger`"
},
),
"interval":
Int(
doc={
"title":
"interval",
"text":
"The single interval, in seconds, that defines the window period"
})
|
Seq(Int(
doc={
def generate_schema(
stage_actions: OptionalType[List[str]] = None,
default_compute_type: OptionalType[str] = None,
default_image: OptionalType[str] = None,
log_group_config: Dict = None,
) -> Map:
"""Generate a schema"""
input_artifact_validator = Str()
if stage_actions:
input_artifact_validator = Enum(stage_actions)
name_validation_key = Optional("name")
if (log_group_config and log_group_config["enabled"]
and not log_group_config["create"]):
# name becomes mandatory if we're not creating it and it's enabled
name_validation_key = "name"
log_group_validator = {
Optional(
"enabled",
default=CODEBUILD_DEFAULTS["log_group"]["enabled"],
):
Bool(),
name_validation_key:
Str(),
Optional(
"create",
default=CODEBUILD_DEFAULTS["log_group"]["create"],
):
Bool(),
Optional("retention"):
Int(),
}
return Map({
"config":
Map({
"s3_bucket":
Str(),
"kms_key_arn":
Str(),
Optional(
"codepipeline",
default=CODEPIPELINE_DEFAULTS,
):
Map({
Optional(
"restart_execution_on_update",
default=CODEPIPELINE_DEFAULTS["restart_execution_on_update"],
):
Bool(),
}),
Optional("codebuild", default=CODEBUILD_DEFAULTS):
Map({
Optional(
"compute_type",
default=CODEBUILD_DEFAULTS["compute_type"],
):
Str(),
Optional(
"image",
default=CODEBUILD_DEFAULTS["image"],
):
Str(),
Optional(
"log_group",
default=CODEBUILD_DEFAULTS["log_group"],
):
Map(log_group_validator),
}),
Optional("iam", default=[]):
EmptyList()
| Seq(
Map({
Optional("Effect", default="Allow"):
Enum(["Allow", "Deny"]),
"Action":
Seq(Str()),
"Resource":
Seq(Str()),
})),
}),
"sources":
Seq(
Map({
"name": UniqueStr(),
"from": Enum(["CodeCommit", "CodeStarConnection"]),
"repository": Str(),
"branch": Str(),
Optional("poll_for_source_changes", default=False): Bool(),
Optional("event_for_source_changes", default=True): Bool(),
Optional("connection_arn"): Str(),
})),
"stages":
Seq(
Map({
"name":
UniqueStr(),
Optional("enabled", default=True):
Bool(),
"actions":
Seq(
Map({
"name":
UniqueStr(),
Optional("category", default="Build"):
Enum(["Build", "Test", "Deploy"]),
Optional("provider", default="CodeBuild"):
Enum(["CodeBuild"]),
Optional("buildspec"):
Str(),
Optional("commands"):
Seq(Str()),
Optional("artifacts"):
Seq(Str()),
Optional(
"compute_type",
default=default_compute_type,
):
Str(),
Optional("image", default=default_image):
Str(),
Optional("environment", default={}):
EmptyDict()
| MapPattern(Str(), Str()),
Optional("input_artifacts", default=[]):
EmptyList()
| Seq(input_artifact_validator),
})),
})),
})
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
python_version=GivenProperty(Str()),
selenium_version=GivenProperty(Str()),
website=GivenProperty(MapPattern(Str(), Str())),
selectors_yml=GivenProperty(Str()),
javascript=GivenProperty(Str()),
setup=GivenProperty(Str()),
code=GivenProperty(Str()),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
)
def __init__(self, keypath, settings):
self.path = keypath
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.state = self.path.gen.joinpath("state")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
self.path.profile = self.path.gen.joinpath("profile")
dirtemplate.DirTemplate(
"webapp", self.path.key / "htmltemplate", self.path.state
).with_vars(javascript=self.given.get("javascript", "")).with_files(
base_html={
filename: {
"content": content
}
for filename, content in self.given.get("website", {}).items()
}).ensure_built()
self.path.state.joinpath("selectors.yml").write_text(
self.given["selectors.yml"])
self.server = (python("-m", "http.server").in_dir(self.path.state /
"webapp").pexpect())
self.server.expect("Serving HTTP on 0.0.0.0")
if not self.path.profile.exists():
self.path.profile.mkdir()
self.python = project_build(self.path, self.given["python version"],
self.given["selenium version"]).bin.python
self.example_py_code = (ExamplePythonCode(
self.python,
self.path.state).with_setup_code(self.given.get(
"setup", "")).with_terminal_size(160, 100).with_long_strings())
@validate(
code=Str(),
will_output=Map({
"in python 2": Str(),
"in python 3": Str()
}) | Str(),
raises=Map({
Optional("type"):
Map({
"in python 2": Str(),
"in python 3": Str()
})
| Str(),
Optional("message"):
Map({
"in python 2": Str(),
"in python 3": Str()
})
| Str(),
}),
in_interpreter=Bool(),
)
def run(self, code, will_output=None, raises=None, in_interpreter=False):
if in_interpreter:
code = "{0}\nprint(repr({1}))".format(
"\n".join(code.strip().split("\n")[:-1]),
code.strip().split("\n")[-1])
to_run = self.example_py_code.with_code(code)
if self.settings.get("cprofile"):
to_run = to_run.with_cprofile(
self.path.profile.joinpath("{0}.dat".format(self.story.slug)))
result = (to_run.expect_exceptions().run()
if raises is not None else to_run.run())
if will_output is not None:
actual_output = "\n".join(
[line.rstrip() for line in result.output.split("\n")])
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self.settings.get("rewrite"):
self.current_step.update(**{"will output": actual_output})
else:
raise
if raises is not None:
differential = False # Difference between python 2 and python 3 output?
exception_type = raises.get("type")
message = raises.get("message")
if exception_type is not None:
if not isinstance(exception_type, str):
differential = True
exception_type = (
exception_type["in python 2"]
if self.given["python version"].startswith("2") else
exception_type["in python 3"])
if message is not None:
if not isinstance(message, str):
differential = True
message = (message["in python 2"]
if self.given["python version"].startswith("2")
else message["in python 3"])
try:
result.exception_was_raised(exception_type, message)
except ExpectedExceptionMessageWasDifferent:
if self.settings.get("rewrite") and not differential:
new_raises = raises.copy()
new_raises["message"] = result.exception.message
self.current_step.update(raises=new_raises)
else:
raise
def do_nothing(self):
pass
def pause(self, message="Pause"):
import IPython
IPython.embed()
def tear_down(self):
self.server.kill(signal.SIGTERM)
self.server.wait()
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
setup=GivenProperty(Str()),
boxname=GivenProperty(Str()),
vmname=GivenProperty(Str()),
issue=GivenProperty(Str()),
files=GivenProperty(MapPattern(Str(), Str())),
python_version=GivenProperty(Str()),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
)
def __init__(self, paths, settings):
self.path = paths
self.settings = settings
def set_up(self):
"""Set up your applications and the test environment."""
self.path.cachestate = self.path.gen.joinpath("cachestate")
self.path.state = self.path.gen.joinpath("state")
self.path.working_dir = self.path.gen.joinpath("working")
self.path.build_path = self.path.gen.joinpath("build_path")
self.path.localsync = self.path.gen.joinpath("local_sync")
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
if self.path.localsync.exists():
self.path.localsync.rmtree(ignore_errors=True)
self.path.localsync.mkdir()
if self.path.build_path.exists():
self.path.build_path.rmtree(ignore_errors=True)
self.path.build_path.mkdir()
self.python = hitchpylibrarytoolkit.project_build(
"hitchbuildvagrant", self.path,
self.given.get("python_version", "3.7.0")).bin.python
if not self.path.cachestate.exists():
self.path.cachestate.mkdir()
for filename, contents in self.given.get("files", {}).items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(contents)
if self.path.working_dir.exists():
self.path.working_dir.rmtree(ignore_errors=True)
self.path.working_dir.mkdir()
self.example_py_code = (ExamplePythonCode(
self.python, self.path.state).with_setup_code(
self.given.get("setup", "").replace(
"/path/to/share",
self.path.cachestate)).with_terminal_size(
160, 100).with_long_strings(
share=str(self.path.cachestate),
build_path=str(self.path.build_path),
issue=str(self.given.get("issue")),
boxname=str(self.given.get("boxname")),
vmname=str(self.given.get("vmname")),
local_sync_path=str(self.path.localsync),
))
@no_stacktrace_for(HitchRunPyException)
def run(self, code):
self.example_py_code.with_code(code).run()
def write_to_localsync(self, **files):
for filename, contents in files.items():
self.path.localsync.joinpath(filename).write_text(contents)
def delete_localsync_file(self, filename):
self.path.localsync.joinpath(filename).remove()
def write_file(self, filename, contents):
self.path.state.joinpath(filename).write_text(contents)
def raises_exception(self, message=None, exception_type=None):
try:
result = self.example_python_code.expect_exceptions().run(
self.path.state, self.python)
result.exception_was_raised(exception_type, message.strip())
except ExpectedExceptionMessageWasDifferent as error:
if self.settings.get("rewrite"):
self.current_step.update(message=error.actual_message)
else:
raise
def file_contains(self, filename, contents):
assert (self.path.working_dir.joinpath(filename).bytes().decode("utf8")
== contents)
@validate(duration=Float())
def sleep(self, duration):
import time
time.sleep(duration)
def pause(self, message="Pause"):
import IPython
IPython.embed()
def on_failure(self, reason):
pass
def tear_down(self):
for vagrantfile in pathquery(self.path.state).named("Vagrantfile"):
Command("vagrant", "destroy",
"-f").in_dir(vagrantfile.abspath().dirname()).run()
Opt("mail"):
Map({
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
}),
})
_job_defaults_common = {
Opt("shell"):
Str(),
Opt("concurrencyPolicy"):
Enum(["Allow", "Forbid", "Replace"]),
Opt("captureStderr"):
Bool(),
Opt("captureStdout"):
Bool(),
Opt("saveLimit"):
Int(),
Opt("utc"):
Bool(),
Opt("failsWhen"):
Map({
"producesStdout": Bool(),
Opt("producesStderr"): Bool(),
Opt("nonzeroReturn"): Bool(),
Opt("always"): Bool(),
}),
"id": Int(),
"name": Str(),
"code" : Str(),
Optional("description"): Str()
})),
"commands": Seq(Map({
"id": Int(),
"name": Str(),
"code": Str(),
Optional("sinceVersion"): Int(),
Optional("tillVersion"): Int(),
Optional("description"): Str(),
"messages": Seq(Map({
Optional("senders") : Seq(Str()),
Optional("receivers") : Seq(Str()),
"messageTypes": UniqueSeq(Enum(['>', '<', '!', '#'])),
Optional("description"): Str(),
"structureType": Enum(['md', 'markdown', 'text']),
"structureDesc": Str()
}))
}))
})
def output_yaml_lines(yaml):
indent = " "
print(indent + "line: " + str(yaml.start_line), file=sys.stderr)
for line in yaml.lines().splitlines():
print(indent*2 + line, file=sys.stderr)
def verify(yaml):
not_ascci_range = re.compile('[^ -~]').search
Opt("level"): Str(),
Opt("extra"): MapPattern(Str(), Str() | Int() | Bool()),
}),
Opt("mail"): Map({
"from": EmptyNone() | Str(),
"to": EmptyNone() | Str(),
Opt("smtpHost"): Str(),
Opt("smtpPort"): Int(),
Opt("subject"): Str(),
Opt("body"): Str(),
})
})
_job_defaults_common = {
Opt("shell"): Str(),
Opt("concurrencyPolicy"): Enum(['Allow', 'Forbid', 'Replace']),
Opt("captureStderr"): Bool(),
Opt("captureStdout"): Bool(),
Opt("saveLimit"): Int(),
Opt("utc"): Bool(),
Opt("failsWhen"): Map({
"producesStdout": Bool(),
Opt("producesStderr"): Bool(),
Opt("nonzeroReturn"): Bool(),
Opt("always"): Bool(),
}),
Opt("onFailure"): Map({
Opt("retry"): Map({
"maximumRetries": Int(),
"initialDelay": Float(),
"maximumDelay": Float(),
"id": id_selector,
"class": class_selector,
"attribute": attribute_selector,
"text is": text_is_selector,
"text contains": text_contains_selector,
"xpath": xpath_selector,
}
ELEMENTS_SCHEMA = MapPattern(
Str(),
Str()
| Map(
{
Optional("in iframe"): Str(),
Optional("which"): Enum(["last"]) | Int(),
Optional("but parent"): Int(),
Optional("subelements"): Any(),
# SELECTORS
Optional("id"): Str(),
Optional("class"): Str(),
Optional("attribute"): Str(),
Optional("text is"): Str(),
Optional("text contains"): Str(),
Optional("xpath"): Str(),
}
),
)
def revalidate_subelements(elements):
from strictyaml import Map, MapPattern, Optional
from strictyaml import Str, Int, Seq, Enum, Any, as_document
JSONSCHEMA_TYPE_SNIPPET = {
"type": Enum(["object", "integer", "string", "array"]),
Optional("required"): Seq(Str()),
Optional("properties"): MapPattern(Str(), Any()),
Optional("items"): Any(),
}
JSONSCHEMA_SCHEMA = Map(JSONSCHEMA_TYPE_SNIPPET)
def get_schema(snippet):
if snippet['type'] == "integer":
return Int()
elif snippet['type'] == "string":
return Str()
elif snippet['type'] == "array":
return Seq(get_schema(snippet["items"]))
elif snippet['type'] == "object":
map_schema = {}
for key, subschema in snippet['properties'].items():
if key in snippet.get('required', []):
map_schema[Optional(key)] = get_schema(subschema)
else:
map_schema[key] = get_schema(subschema)
return Map(map_schema)
def load_schema(json_schema):
class Engine(BaseEngine):
"""Python engine for running tests."""
given_definition = GivenDefinition(
python_version=GivenProperty(Str()),
files=GivenProperty(MapPattern(Str(), Str())),
)
info_definition = InfoDefinition(
status=InfoProperty(schema=Enum(["experimental", "stable"])),
docs=InfoProperty(schema=Str()),
fails_on_python_2=InfoProperty(schema=Bool()),
description=InfoProperty(schema=Str()),
experimental=InfoProperty(schema=Bool()),
)
def __init__(self, keypath, rewrite=False, build=False):
self.path = keypath
self._rewrite = rewrite
self._build = hitchpylibrarytoolkit.PyLibraryBuild(
"hitchqs",
self.path,
)
def set_up(self):
"""Set up your applications and the test environment."""
self._build.ensure_built()
self.path.state = self.path.project.parent / "tempqs"
if self.path.state.exists():
self.path.state.rmtree(ignore_errors=True)
self.path.state.mkdir()
for filename, contents in self.given.get("files", {}).items():
filepath = self.path.state.joinpath(filename)
if not filepath.dirname().exists():
filepath.dirname().makedirs()
filepath.write_text(contents)
self.path.profile = self.path.gen.joinpath("profile")
if not self.path.profile.exists():
self.path.profile.mkdir()
self.qs = self._build.bin.quickstart
def _run(self, command, args, will_output=None, exit_code=0, timeout=5):
process = command(*shlex.split(args)).interact().screensize(160, 80).run()
process.wait_for_finish()
actual_output = process.stripshot()\
.replace(self.path.state, "/path/to")
# Replace 35.1 SECONDS with n.n SECONDS
actual_output = re.sub("[0-9]+\.[0-9]", "n.n", actual_output)
if will_output is not None:
try:
Templex(will_output).assert_match(actual_output)
except AssertionError:
if self._rewrite:
self.current_step.update(**{"will output": actual_output})
else:
raise
assert process.exit_code == exit_code, "Exit code should be {} was {}, output:\n{}".format(
exit_code,
process.exit_code,
actual_output,
)
@validate(timeout=Int(), exit_code=Int())
@no_stacktrace_for(AssertionError)
def quickstart(self, args, will_output=None, exit_code=0, timeout=5):
self._run(self.qs.in_dir(self.path.state), args, will_output, exit_code, timeout=timeout)
@no_stacktrace_for(AssertionError)
def hk(self, args, will_output=None, exit_code=0, timeout=5, in_dir=""):
if self._build:
self._run(Command("hk").in_dir(self.path.state / in_dir), args, will_output, exit_code, timeout=timeout)
def initial_hk(self, args="", in_dir=""):
if self._build:
Command("hk", *shlex.split(args)).in_dir(self.path.state / in_dir).run()
@validate(filenames=Seq(Str()))
def files_appear(self, filenames):
appeared = set()
should_appear = set(filenames)
for existing_file in pathquery(self.path.state):
if "__pycache__" not in existing_file and not existing_file.isdir():
appeared.add(str(existing_file.relpath(self.path.state)))
diff = should_appear.symmetric_difference(appeared)
assert diff == set(), \
"Difference in files that appeared:\n{}".format('\n'.join(diff))
def pause(self, message="Pause"):
import IPython
IPython.embed()
def tear_down(self):
if self._build:
if self.path.state.exists():
Command("hk", "--clean").ignore_errors().in_dir(self.path.state).run()