def run(self, data, config=None, pipeline=None):
"""Replace results of `self.select_expr` with `self.value_expr`."""
opts = jmespath.Options(custom_functions=JMESExtensions(data))
to_replace = jmespath.search(self.select_expr, data, opts)
replace_with = jmespath.search(self.value_expr, to_replace, opts)
if isinstance(to_replace, collections.abc.Mapping):
if not isinstance(replace_with, collections.abc.Mapping):
self._type_error(to_replace, replace_with)
# Edge case: If the two elements are the same, then we cannot
# call "clear", because it will erase both. In that case, do
# nothing.
if to_replace is not replace_with:
to_replace.clear()
to_replace.update(replace_with)
elif isinstance(to_replace, collections.abc.Sequence):
if not isinstance(replace_with, collections.abc.Sequence):
self._type_error(to_replace, replace_with)
self.logger.info(
("Replacing result of '%s' (a list of length %d) "
"with a %d-element list"),
self.select_expr, len(to_replace), len(replace_with))
to_replace[:] = replace_with
else:
self._invalid_selection(to_replace)
return data
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = None
try:
actual_result = jmespath.search(
self._query, json_value,
jmespath.Options(collections.OrderedDict))
except jmespath.exceptions.JMESPathTypeError:
raise JMESPathCheckAssertionError(self._query,
self._expected_result,
actual_result,
execution_result.output)
if self._case_sensitive:
equals = actual_result == self._expected_result or str(
actual_result) == str(self._expected_result)
else:
equals = actual_result == self._expected_result \
or str(actual_result).lower() == str(self._expected_result).lower()
if not equals:
if actual_result:
raise JMESPathCheckAssertionError(self._query,
self._expected_result,
actual_result,
execution_result.output)
raise JMESPathCheckAssertionError(self._query,
self._expected_result, 'None',
execution_result.output)
def run(self, data, config=None, pipeline=None):
"""Collect and index the files that form an hhsuite database."""
jmes_opts = jmespath.Options(custom_functions=JMESExtensions(data))
templates = jmespath.search(self.select_expr, data, jmes_opts)
with tempfile.NamedTemporaryFile("w") as file_list:
# Write IDs to remove to temp file
for template in templates:
print(template["name"], file=file_list)
file_list.flush()
db_types = ["a3m", "hhm", "cs219"]
db_prefix = Path(self.db_prefix)
for file_type in db_types:
ffindex = Path("{}_{}.ffindex".format(db_prefix, file_type))
# if ffindex.exists():
cmd_line = tools.ffindex_modify(
(self.bin_dir, "ffindex_modify"),
options={"file_list": file_list.name},
flags=["sort", "unlink"],
positional=[ffindex])
self.logger.debug("Running command %s", cmd_line)
tools.run(cmd_line, check=True)
return data
def _get_hashed_idempotency_key(self, event: Dict[str, Any],
context: LambdaContext) -> str:
"""
Extract data from lambda event using event key jmespath, and return a hashed representation
Parameters
----------
event: Dict[str, Any]
Lambda event
context: LambdaContext
Lambda context
Returns
-------
str
Hashed representation of the data extracted by the jmespath expression
"""
data = event
if self.event_key_jmespath:
data = self.event_key_compiled_jmespath.search(
event, options=jmespath.Options(**self.jmespath_options))
if self.is_missing_idempotency_key(data):
if self.raise_on_no_idempotency_key:
raise IdempotencyKeyError(
"No data found to create a hashed idempotency_key")
warnings.warn(
f"No value found for idempotency_key. jmespath: {self.event_key_jmespath}"
)
generated_hash = self._generate_hash(data)
return f"{context.function_name}#{generated_hash}"
def run(self, data, config=None, pipeline=None):
"""Parse metadata from mmCIF file."""
pdb_id = self.get_vals(data)
mmcif_file = phyre_engine.tools.pdb.find_pdb(pdb_id,
suffix_list=(".cif",
".cif.gz"),
base_dir=self.mmcif_dir)
data.setdefault("metadata", {})
if mmcif_file is None:
raise FileNotFoundError(
"Could not find mmCIF file {} in {}".format(
pdb_id, self.mmcif_dir))
with phyre_engine.tools.pdb.open_pdb(mmcif_file) as mmcif_in:
if self.prefilter:
mmcif_in = self._prefilter(mmcif_in)
mmcif_dict = Bio.PDB.MMCIF2Dict.MMCIF2Dict(mmcif_in)
jmes_extensions = JMESExtensions(mmcif_dict)
jmes_opts = jmespath.Options(custom_functions=jmes_extensions)
for field, jmespath_expr in self.fields.items():
value = jmespath.search(jmespath_expr, mmcif_dict, jmes_opts)
data["metadata"][field] = value
return data
def unwrap_event_from_envelope(data: Union[Dict, str], envelope: str,
jmespath_options: Optional[Dict]) -> Any:
"""Searches data using JMESPath expression
Parameters
----------
data : Dict
Data set to be filtered
envelope : str
JMESPath expression to filter data against
jmespath_options : Dict
Alternative JMESPath options to be included when filtering expr
Returns
-------
Any
Data found using JMESPath expression given in envelope
"""
if not jmespath_options:
jmespath_options = {"custom_functions": PowertoolsFunctions()}
try:
logger.debug(
f"Envelope detected: {envelope}. JMESPath options: {jmespath_options}"
)
return jmespath.search(envelope,
data,
options=jmespath.Options(**jmespath_options))
except (LexerError, TypeError, UnicodeError) as e:
message = f"Failed to unwrap event from envelope using expression. Error: {e} Exp: {envelope}, Data: {data}" # noqa: B306, E501
raise InvalidEnvelopeExpressionError(message)
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = jmespath.search(self._query, json_value,
jmespath.Options(collections.OrderedDict))
if actual_result:
raise JMESPathCheckAssertionError(self._query, 'some value', actual_result,
execution_result.output)
def logs(obj, api_id, line, output):
"""
Get API logs
"""
api_client = obj['api_client']
(logs, _, to_timestamp) = api_client.logs(api_id,
line,
time_frame_seconds=parse("1d"))
query = "[].{Date: datetime(timestamp,'%d-%m-%Y %H:%M:%S %f'), App: application_name, verbe: upper(method), Status: status, Path: path, Latency: responseTime}"
try:
logs_filtered = jmespath.search(
query, logs['logs'],
jmespath.Options(custom_functions=GioFunctions()))
if logs_filtered and len(logs_filtered) > 0:
if type(logs_filtered) is list and type(
logs_filtered[0]) is dict and len(logs_filtered) > 0:
header = logs_filtered[0].keys()
OutputFormatType.value_of(output).echo(logs_filtered,
header=header)
else:
click.echo("No logs")
except exceptions.JMESPathError as jmespatherr:
logging.exception("LIST JMESPathError exception")
raise GraviteeioError(str(jmespatherr))
def key_fn(datum):
"""Getter closure using `jmespath_key`."""
jmespath_opts = jmespath.Options(custom_functions=JMESExtensions(root))
field_value = jmespath.search(jmespath_key, datum, jmespath_opts)
if allow_none:
return (field_value is None, field_value)
return field_value
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = jmespath.search(self._query, json_value,
jmespath.Options(collections.OrderedDict))
if not re.match(self._expected_result, str(actual_result), re.IGNORECASE):
raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result,
execution_result.output)
def _filter(self, data):
"""Filter data to contain the required fields."""
if self.select_expr is None:
return data
opts = jmespath.Options(custom_functions=JMESExtensions(data))
return jmespath.search(self.select_expr, data, opts)
def health(obj, output, query, api_id):
"""API health: return the lastest availability: minute, hour, day, week, month"""
api_client: ApiClient = obj['api_client']
health_values = None
# health = api_client.health(api_id, time_frame_seconds = parse(time_frame))
health = api_client.health(api_id)
if 'global' in health:
health_values = health['global']
else:
click.echo("No health data")
return
to_return = []
for key, value in health_values.items():
to_return.append({"time": key, "percent": round(value, 2)})
try:
health_filtered = jmespath.search(
query, to_return,
jmespath.Options(custom_functions=GioFunctions()))
header = None
if len(health_filtered) > 0 and type(health_filtered[0]) is dict:
header = health_filtered[0].keys()
outputFormat = OutputFormatType.value_of(output)
outputFormat.echo(health_filtered, header=header)
except exceptions.JMESPathError as jmespatherr:
logging.exception("HEALTH JMESPathError exception")
raise GraviteeioError(str(jmespatherr))
except Exception:
logging.exception("HEALTH Exception")
raise GraviteeioError("to print {} with the format {}.".format(
health_filtered, format))
def compare(self, json_data):
if not json_data:
json_data = '{}'
json_val = json.loads(json_data)
actual_result = jmespath.search(
self.query, json_val, jmespath.Options(collections.OrderedDict))
if not actual_result == self.expected_result:
raise JMESPathCheckAssertionError(self, actual_result, json_data)
def _search_result_by_jmespath(json_data, query):
if not json_data:
json_data = '{}'
json_val = json.loads(json_data)
return jmespath.search(
query,
json_val,
jmespath.Options(collections.OrderedDict))
def jmespath_init():
class CustomFunctions(functions.Functions):
@functions.signature({'types': ['number']})
def _func_isotime(self, x):
d = datetime.fromtimestamp(x)
return d.isoformat()
options = jmespath.Options(custom_functions=CustomFunctions())
return options
def get_value(self, data):
"Extracts data from a data dict."
if self.omit:
return None
if self.value:
return self.value
if self.expression:
options = jmespath.Options(custom_functions=Functions())
return jmespath.search(self.expression, data, options=options)
return data.get(self.field_name)
def test_can_provide_dict_cls(self):
result = jmespath.search(
'{a: a, b: b, c: c}.*', {
'c': 'c',
'b': 'b',
'a': 'a',
'd': 'd'
},
options=jmespath.Options(dict_cls=OrderedDict))
self.assertEqual(result, ['a', 'b', 'c'])
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = jmespath.search(self._query, json_value,
jmespath.Options(collections.OrderedDict))
if not actual_result == self._expected_result:
if actual_result:
raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result,
execution_result.output)
else:
raise JMESPathCheckAssertionError(self._query, self._expected_result, 'None',
execution_result.output)
def search(expression, data):
"""
自定义search方法，自动传入options参数
同时规避这个bug https://github.com/jmespath/jmespath.py/issues/133 v0.9.3已修复
:param expression:
:param data:
:return:
"""
options = jmespath.Options(custom_functions=Functions())
# 使用自定义方法后，所有操作都需要带上options参数
return jmespath.search(expression, data, options)
def generate_config(self, data, config):
"""
Generate child pipeline configuration from runtime configuration
and pipeline state.
"""
config = config if config is not None else {}
jmes_opts = jmespath.Options(custom_functions=JMESExtensions(data))
for search_term, config_location in self.mapping.items():
state_value = jmespath.search(search_term, data, jmes_opts)
apply_dotted_key(config, config_location, state_value)
return config
def run(self, data, config=None, pipeline=None):
"""Run em4gmm for automatic clustering."""
# Select sample
jmes_opts = jmespath.Options(custom_functions=JMESExtensions(data))
sample_list = jmespath.search(self.select_expr, data, jmes_opts)
# Extract data points
data_points = [
jmespath.search(self.dimensions_expr, sample, jmes_opts)
for sample in sample_list
]
num_samples = len(data_points)
num_dims = len(data_points[0])
# Write samples to file
with tempfile.NamedTemporaryFile("w") as sample_file:
print("{} {}".format(num_dims, num_samples), file=sample_file)
for sample in data_points:
print(" ".join([str(i) for i in sample]), file=sample_file)
sample_file.flush()
# Run trainer
gmmtrain_opts = {"samples": sample_file.name}
gmmtrain_opts.update(self.gmmtrain_opts)
gmmtrain = self.GMMTRAIN((self.bin_dir, "gmmtrain"),
options=gmmtrain_opts)
self.logger.debug("Running %s", gmmtrain)
subprocess.run(gmmtrain, check=True)
# Run classifier
gmmclass_opts = {"samples": sample_file.name}
gmmclass_opts.update(self.gmmclass_opts)
gmmclass = self.GMMCLASS((self.bin_dir, "gmmclass"),
options=gmmclass_opts)
self.logger.debug("Running %s", gmmclass)
subprocess.run(gmmclass, check=True)
# Parse cluster definitions from trainer log file
with open(self.gmmtrain_opts["model_details"], "r") as model_in:
model = json.load(model_in)
data["clusters"] = model
# Parse sample data, adding to the samples
with open(self.gmmclass_opts["sample_details"], "r") as samples_in:
sample_details = json.load(samples_in)["samples_results"]
for details in sample_details:
i = details["sample"]
sample_list[i]["cluster"] = details["class"]
sample_list[i]["lprob"] = details["lprob"]
return data
def status(obj, output, query, time_frame, api_id):
"""
Displays status of API.
\b
Status Field:
- `status`: string
- `hits`: numerate
- `percent`: string
"""
api_client: ApiClient = obj['api_client']
try:
status_values = api_client.status(api_id, parse(time_frame))['values']
except TypeError:
raise GraviteeioError("Unsupported type for time frame")
to_return = []
total = 0
for key, value in status_values.items():
status_str = "{}xx".format(key[0:1])
to_return.append({
"status": status_str,
"hits": value,
"percent": value * 100
})
total += value
if total > 0:
for status in to_return:
status["percent"] = round(status["percent"] / total, 2)
if format == "hbar":
query = "reverse(@)[].{Status: status, Percent: percent}"
# start_time = time.time()
try:
status_filtered = jmespath.search(
query, to_return,
jmespath.Options(custom_functions=GioFunctions()))
header = None
if len(status_filtered) > 0 and type(status_filtered[0]) is dict:
header = status_filtered[0].keys()
OutputFormatType.value_of(output).echo(status_filtered, header=header)
except exceptions.JMESPathError as jmespatherr:
logging.exception("STATUS JMESPathError exception")
raise GraviteeioError(str(jmespatherr))
except Exception:
logging.exception("STATUS Exception")
raise GraviteeioError("to print {} with the format {}.".format(
status_filtered, format))
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
if not json_value:
raise AssertionError(
'The command output cannot be parsed in json.')
actual_result = jmespath.search(
self._query, json_value, jmespath.Options(collections.OrderedDict))
if not actual_result:
raise JMESPathCheckAssertionError(self._query, 'some value',
actual_result,
execution_result.output)
def json_query(data, expr):
'''Query data using jmespath query language ( http://jmespath.org ). Example:
- debug: msg="{{ instance | json_query(tagged_instances[*].block_device_mapping.*.volume_id') }}"
'''
if not HAS_LIB:
raise AnsibleError('You need to install "jmespath" prior to running '
'json_query filter')
jmespath.functions.REVERSE_TYPES_MAP[
'string'] = jmespath.functions.REVERSE_TYPES_MAP['string'] + (
'AnsibleUnicode', 'AnsibleUnsafeText')
options = jmespath.Options(custom_functions=CustomFunctions())
return jmespath.search(expr, data, options=options)
def run(self, data, config=None, pipeline=None):
"""Collect and index the files that form an hhsuite database."""
jmes_opts = jmespath.Options(custom_functions=JMESExtensions(data))
templates = jmespath.search(self.select_expr, data, jmes_opts)
# First, sort templates by sequence length.
templates.sort(key=lambda t: len(t["sequence"]))
# Make database directory if it doesn't exist.
Path(self.db_prefix).parent.mkdir(parents=True, exist_ok=True)
# Collect a3m/hhm/cs219 files into ffindex/ffdata databases
to_collect = ["a3m", "hhm", "cs219"]
ff_dbs = {}
db_prefix = Path(self.db_prefix)
for file_type in to_collect:
db_name = Path("{}_{}".format(str(db_prefix), file_type))
ffindex = Path("{}.ffindex".format(str(db_name)))
ffdata = Path("{}.ffdata".format(str(db_name)))
if self.overwrite:
if ffindex.exists():
ffindex.unlink()
if ffdata.exists():
ffdata.unlink()
with tempfile.NamedTemporaryFile("w") as index:
# Write all files of file_type `file_type` to a temp file
for template in templates:
print(template[file_type], file=index)
index.flush()
# Run ffindex_build using the the temp file as the list of files
# to include in the DB.
cmd_line = tools.ffindex_build(
(self.bin_dir, "ffindex_build"),
positional=[ffdata, ffindex],
flags=["sort", "append"],
options={"file_list": index.name})
self.logger.debug("Running command %s", cmd_line)
tools.run(cmd_line, check=True)
ff_dbs[file_type] = db_name
# Cut useless information from the indices of each file.
for ff_db in ff_dbs.values():
self._trim_index_names(ff_db)
data["database"] = str(db_prefix)
return data
def __call__(self, execution_result):
json_value = execution_result.get_output_in_json()
actual_result = None
try:
actual_result = jmespath.search(self._query, json_value,
jmespath.Options(collections.OrderedDict))
except jmespath.exceptions.JMESPathTypeError:
raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result,
execution_result.output)
if actual_result != self._expected_result and str(actual_result) != str(self._expected_result):
if actual_result:
raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result,
execution_result.output)
else:
raise JMESPathCheckAssertionError(self._query, self._expected_result, 'None',
execution_result.output)
def section(self, section_expr):
"""
Retrieve a section of the pipeline state by evaluating the JMESPath
expression `section_expr`.
:param str section_expr: JMESPath expression returning a section of the
pipeline configuration.
:returns: Section of the pipeline configuration.
:rtype: :py:class:`.PipelineConfig`
"""
jmes_ext = phyre_engine.tools.jmespath.JMESExtensions(self)
jmes_opts = jmespath.Options(custom_functions=jmes_ext)
section = jmespath.search(section_expr, self, jmes_opts)
if section is None:
return type(self)()
return type(self)(section)
def test_can_provide_custom_functions(self):
class CustomFunctions(jmespath.functions.Functions):
@jmespath.functions.signature({'types': ['number']},
{'types': ['number']})
def _func_custom_add(self, x, y):
return x + y
@jmespath.functions.signature({'types': ['number']},
{'types': ['number']})
def _func_my_subtract(self, x, y):
return x - y
options = jmespath.Options(custom_functions=CustomFunctions())
self.assertEqual(
jmespath.search('custom_add(`1`, `2`)', {}, options=options), 3)
self.assertEqual(
jmespath.search('my_subtract(`10`, `3`)', {}, options=options), 7)
def run(self, data, config=None, pipeline=None):
"""Sort pipeline state."""
jmespath_opts = jmespath.Options(custom_functions=JMESExtensions(data))
to_sort = jmespath.search(self.field, data, jmespath_opts)
# Sort according to each key, running from last to first to take
# advantage of Python's stable sorting.
for sort_key in reversed(self.keys):
reverse = sort_key.get("reverse", False)
allow_none = sort_key.get("allow_none", False)
to_sort = jmes_sort(to_sort,
sort_key["key"],
root=data,
reverse=reverse,
allow_none=allow_none)
jmespath.search(self.field, data, jmespath_opts)[:] = to_sort
return data
def print_result(result, jmespath_expr=None, format_function=None):
""" Print result vule """
format_function = format_function or helpers.json_dumps
# Query using jmespath
if jmespath_expr is not None:
keep_dict_order = jmespath.Options(dict_cls=OrderedDict)
result = jmespath_expr.search(result, keep_dict_order)
# Format output
formatted = format_function(result)
try:
print(formatted)
except IOError as err:
# Ignore BrokenPipe
if err.errno != errno.EPIPE:
raise err