def value_error_is_raised_if_program_is_invalid():
try:
jq.compile("!")
assert False, "Expected error"
except ValueError as error:
expected_error_str = "jq: error: syntax error, unexpected INVALID_CHARACTER, expecting $end (Unix shell quoting issues?) at <top-level>, line 1:\n!\njq: 1 compile error"
assert_equal(str(error), expected_error_str)
def update(self, response) -> None:
log.MainLogger().debug("Komponente "+self.component_config["name"]+" auslesen.")
config = self.component_config["configuration"]
power = float(jq.compile(config["jq_power"]).input(response).first())
if power >= 0:
power = power * -1
if config["jq_counter"] == "":
topic_str = "openWB/set/system/device/" + \
str(self.__device_id)+"/component/" + \
str(self.component_config["id"])+"/"
_, counter = self.__sim_count.sim_count(
power, topic=topic_str, data=self.simulation, prefix="pv")
inverter_state = InverterState(
power=power,
counter=counter,
currents=[0, 0, 0]
)
else:
counter = jq.compile(config["jq_counter"]).input(response).first()
inverter_state = InverterState(
power=power,
counter=counter
)
self.__store.set(inverter_state)
def update(self, response) -> None:
config = self.component_config.configuration
power = jq.compile(config.jq_power).input(response).first()
if config.jq_soc != "":
soc = jq.compile(config.jq_soc).input(response).first()
else:
soc = 0
if config.jq_imported != "" and config.jq_exported != "":
imported = jq.compile(config.jq_imported).input(response).first()
exported = jq.compile(config.jq_exported).input(response).first()
else:
topic_str = "openWB/set/system/device/" + str(
self.__device_id)+"/component/"+str(self.component_config.id)+"/"
imported, exported = self.__sim_count.sim_count(
power, topic=topic_str, data=self.simulation, prefix="speicher"
)
bat_state = BatState(
power=power,
soc=soc,
imported=imported,
exported=exported
)
self.__store.set(bat_state)
def update(self, response):
log.MainLogger().debug("Komponente "+self.component_config["name"]+" auslesen.")
config = self.component_config["configuration"]
power = jq.compile(config["jq_power"]).input(response).first()
if config["jq_imported"] == "" or config["jq_exported"] == "":
topic_str = "openWB/set/system/device/{}/component/{}/".format(
self.__device_id, self.component_config["id"]
)
imported, exported = self.__sim_count.sim_count(
power,
topic=topic_str,
data=self.simulation,
prefix="bezug"
)
else:
imported = jq.compile(config["jq_imported"]).input(response).first()
exported = jq.compile(config["jq_exported"]).input(response).first()
counter_state = CounterState(
imported=imported,
exported=exported,
power=power
)
self.__store.set(counter_state)
def delete_comments(repo, pull_number):
print('clearing comments')
bot = 'github-actions[bot]'
comment_url = ISSUE_COMMENTS.format(repo=repo, issue_number=pull_number)
comments = request('get', comment_url)
jq_user = jq.compile('.user.login')
jq_comment = jq.compile('.id')
for comment in comments:
user = jq_user.input(comment).first()
if user == bot:
comment_id = jq_comment.input(comment).first()
delete_comment(repo, comment_id)
def compiling_with_args_should_set_predefined_variables():
assert_equal(
123,
jq.compile("$a + $b + .", args={
"a": 100,
"b": 20
}).input(3).first())
async def locationsv1_get(
db: DB = Depends(),
locations: Locations = Depends(Locations.depends()),
):
data = await locations_get(db, locations)
meta = data.meta
res = data.results
latest_jq = jq.compile("""
.[] |
{
id: .id,
country: .country,
city: .city,
location: .name,
soureName: .source_name,
sourceType: .sources[0].name,
coordinates: .coordinates,
firstUpdated: .firstUpdated,
lastUpdated: .lastUpdated,
parameters : [ .parameters[].parameter ],
countsByMeasurement: [
.parameters[] | {
parameter: .parameter,
count: .count
}
],
count: .parameters| map(.count) | add
}
""")
ret = latest_jq.input(res).all()
return OpenAQResult(meta=meta, results=ret)
def program_preserves_floats():
program = jq.compile(".")
assert_equal(float, type(program.input(text="1.1").first()))
assert_equal(3.14159, program.input(text="3.14159").first())
assert_equal(-3.14159, program.input(text="-3.14159").first())
assert_equal(42E100, program.input(text="42E100").first())
assert_equal(-42E100, program.input(text="-42E100").first())
def jq_select(expression: str, obj):
if has_jq:
return jq.compile(expression).input(obj).all()
else:
return json.loads(
subprocess.check_output(['jq', expression],
input=json.dumps(obj).encode()))
def main():
"""Execute PY_SETUP (if set) and call PY_TARGET on JSON from stdin. See
https://github.com/wbadart/rmq_py_caller#readme for details.
"""
parser = ArgumentParser(prog="python -m rmq_py_caller",
description=main.__doc__)
parser.add_argument(
"-v",
"--verbose",
action="count",
default=0,
help="-v to include info, -vv to include debugging",
)
args = parser.parse_args()
# If you're curious about the use exec/ eval here, please see the project
# wiki for discussion.
# pylint: disable=exec-used,eval-used
exec(environ.get("PY_SETUP", ""), globals())
ctx = eval(environ["PY_TARGET"])
# Wait until now to call basicConfig in case PY_SETUP wants to do it
if args.verbose:
logging.basicConfig(
level=[logging.WARN, logging.INFO, logging.DEBUG][args.verbose])
log = logging.getLogger(__name__)
log.debug("Got PY_TARGET: %s", ctx)
adapter = jq.compile(environ.get("ARG_ADAPTER", "[.]"))
log.debug("Using ARG_ADAPTER: %s", adapter.program_string)
main_loop(ctx, adapter, sys.stdin)
async def latest_get(
db: DB = Depends(),
locations: Locations = Depends(Locations.depends()),
):
data = await locations_get(db, locations)
meta = data.meta
res = data.results
if len(res) == 0:
return data
latest_jq = jq.compile("""
.[] |
{
location: .name,
city: .city,
country: .country,
coordinates: .coordinates,
measurements: [
.parameters[] | {
parameter: .measurand,
value: .lastValue,
lastUpdated: .lastUpdated,
unit: .unit
}
]
}
""")
ret = latest_jq.input(res).all()
return OpenAQResult(meta=meta, results=ret)
def program_preserves_arrays():
program = jq.compile(".")
assert_equal(list, type(program.input(text='[]').first()))
assert_equal([], program.input(text='[]').first())
assert_equal([1], program.input(text='[1]').first())
assert_equal([1, 2], program.input(text='[1, 2]').first())
assert_equal([3.14159], program.input(text='[3.14159]').first())
assert_equal([3.14159, 95141.3],
program.input(text='[3.14159, 95141.3]').first())
assert_equal([False], program.input(text='[false]').first())
assert_equal([False, True], program.input(text='[false, true]').first())
assert_equal([[]], program.input(text='[[]]').first())
assert_equal([[[]]], program.input(text='[[[]]]').first())
assert_equal([[], []], program.input(text='[[], []]').first())
assert_equal([[[], []], [[], []]],
program.input(text='[[[], []], [[], []]]').first())
assert_equal([{}], program.input(text='[{}]').first())
assert_equal([{"": []}], program.input(text='[{"": []}]').first())
assert_equal([{
"1": [],
"2": []
}],
program.input(text='[{"1": [], "2": []}]').first())
def ps_hook(tasking: models.Tasking):
"""
This hook watches for the 'ps' command and writes the processes into the processes table.
For Powershell Agents, the data comes back (as of 4.1) as JSON.
For Python Agents, the data comes back in the typical 'ls' format.
For C# Agents, no support yet.
AFAIK, it is not easy to convert the shell tables into JSON, but I found this jq wizardry
on StackOverflow, so that's what we'll stick with for now for the python results, even though it is imperfect.
https://unix.stackexchange.com/a/243485
"""
if (tasking.input.strip() not in ["ps", "tasklist"]
or tasking.agent.language == "csharp"):
return
if tasking.agent.language == "python":
output = (jq.compile(
"""[sub("\n$";"") | splits("\n") | sub("^ +";"") | [splits(" +")]] | .[0] as $header | .[1:] | [.[] | [. as $x | range($header | length) | {"key": $header[.], "value": $x[.]}] | from_entries]"""
).input(
tasking.output.decode("utf-8").split(
"\r\n ..Command execution completed.")[0]).first())
else:
output = json.loads(tasking.output)
existing_processes = (Session().query(
models.HostProcess.process_id).filter(
models.HostProcess.host_id == tasking.agent.host_id).all())
existing_processes = list(map(lambda p: p[0], existing_processes))
for process in output:
process_name = process.get("CMD") or process.get("ProcessName") or ""
process_id = process.get("PID")
arch = process.get("Arch")
user = process.get("UserName")
if process_id:
# and process_id.isnumeric():
if int(process_id) not in existing_processes:
Session().add(
models.HostProcess(
host_id=tasking.agent.host_id,
process_id=process_id,
process_name=process_name,
architecture=arch,
user=user,
))
elif int(process_id) in existing_processes:
db_process: models.HostProcess = (Session().query(
models.HostProcess).filter(
and_(
models.HostProcess.host_id ==
tasking.agent.host_id,
models.HostProcess.process_id == process_id,
)).first())
if not db_process.agent:
db_process.architecture = arch
db_process.process_name = process_name
Session().commit()
def value_error_is_raised_if_input_is_not_valid_json():
program = jq.compile(".x")
try:
program.input(text="!!").first()
assert False, "Expected error"
except jq.JSONParseError as error:
expected_error_str = "Invalid numeric literal at EOF at line 1, column 2"
assert_equal(str(error), expected_error_str)
def value_error_is_raised_if_input_cannot_be_processed_by_program():
program = jq.compile(".x")
try:
program.input(1).all()
assert False, "Expected error"
except ValueError as error:
expected_error_str = "Cannot index number with string \"x\""
assert_equal(str(error), expected_error_str)
def getAndUploadSettings(project):
"""
Use CircleCI API and jq to pull down a project's Settings
from OnPrem and upload them to the active SaaS project.
"""
settings = compile('."feature_flags" | del(."builds-service") | del(."fleet")').input(getSettings(project)).text()
print('Successfully got settings for project {}:\n{}'.format(project, settings))
uploadSettings(settings)
def test_resolve_employee_detail(self, user, api_client, employee):
api_client.force_authenticate(user)
response = api_client.get(reverse('employee-detail',
args=(1, ))).json()
jq_filtered_response = jq.compile(
'. as $data | {name: $data.name}').input(response).first()
expected = {'name': 'John'}
assert response == expected or jq_filtered_response == expected
def program_preserves_ints():
program = jq.compile(".")
assert_equal(int, type(program.input(text="0").first()))
assert_equal(0, program.input(text="0").first())
assert_equal(1, program.input(text="1").first())
assert_equal(-1, program.input(text="-1").first())
assert_equal(12345, program.input(text="12345").first())
assert_equal(-12345, program.input(text="-12345").first())
def errors_do_not_leak_between_transformations():
program = jq.compile(".x")
try:
program.input(1).all()
assert False, "Expected error"
except ValueError as error:
pass
assert_equal(1, program.input({"x": 1}).first())
def __init__(self, **kwargs):
self.config = kwargs['config']
self.log = get_logger(name=__class__.__name__)
self.cells = defaultdict(list)
self.sort_results = dict()
if len(self.config.sort_queries) > 0:
self.sort_queries = self.config.sort_queries
else:
self.sort_queries = [ jq.compile('.id') ]
def update(self, response):
config = self.component_config.configuration
power = jq.compile(config.jq_power).input(response).first()
# ToDo: add current or power per phase
if config.jq_imported == "" or config.jq_exported == "":
topic_str = "openWB/set/system/device/{}/component/{}/".format(
self.__device_id, self.component_config.id)
imported, exported = self.__sim_count.sim_count(
power, topic=topic_str, data=self.simulation, prefix="bezug")
else:
imported = jq.compile(config.jq_imported).input(response).first()
exported = jq.compile(config.jq_exported).input(response).first()
counter_state = CounterState(imported=imported,
exported=exported,
power=power)
self.__store.set(counter_state)
def filter_copy(source, filter, size=500, use_float=False):
"""Permits response nested key filtering.
Args:
source (obj): Raw server response to filter.
filter (obj): Whitelist filter(s) to apply to the source data. Must be a
:py:class:`list` to use the native DSL, or a string to use a JQ script.
This represents data you want to explicitly keep.
size (int, optional): Array buffer size. (default: ``500``)
Notes:
The source data must not be pre-processed by a JSON converter.
The native filtering DSL is simpler than JQ script, and more human readable.
JQ scripts are executed per item returned, not against the entire JSON as
a whole. JQ scripts provide significantly more features than the native DSL
at the cost of performance.
"""
import ijson
if isinstance(filter, str):
import jq
_jq = True
use_float = True
elif isinstance(filter, list):
_jq = False
def apply(_s, _f):
_out = {}
for i in _f:
x = _filter(_s, i, _out)
_out = x if x else _out
return _out
out = [None] * size
idx = 0
if not _jq:
# to remove ijson BytesIO warning
for x in ijson.items(BytesIO(source.encode('utf-8')),
'item',
use_float=use_float):
if idx >= len(out):
out.extend([None] * size)
out[idx] = deepcopy(apply(x, filter))
idx += 1
del out[idx:]
return out
else:
return deepcopy(jq.compile(filter).input(text=source).first())
def iterators_from_same_program_and_input_are_independent():
program_with_input = jq.compile(".[]+1").input([1, 2, 3])
first = iter(program_with_input)
assert_equal(2, next(first))
second = iter(program_with_input)
assert_equal(2, next(second))
assert_equal(3, next(first))
assert_equal(4, next(first))
assert_equal(3, next(second))
assert_equal(4, next(second))
def post_process_config(config: TConfig,
args: ConfigPostProcessFlags) -> TConfig:
"""Post-processes a configuration according to the given flags."""
config_type = type(config)
config = config.to_dict()
for jq_transform in args.jq_transform:
config = jq.compile(jq_transform).input(config).first()
config = config_type.from_dict(config)
str_template_subs = configuration.parse_template_mapping(args.string_def)
config = configuration.replace_templates(config, str_template_subs)
return config
def update(self, response) -> None:
config = self.component_config.configuration
power = float(jq.compile(config.jq_power).input(response).first())
if power >= 0:
power = power * -1
if config.jq_exported == "":
topic_str = "openWB/set/system/device/" + \
str(self.__device_id)+"/component/" + \
str(self.component_config.id)+"/"
_, exported = self.__sim_count.sim_count(
power,
topic=topic_str,
data=self.simulation,
prefix="pv%s" % ("" if self.component_config.id == 1 else "2"))
else:
exported = jq.compile(config.jq_exported).input(response).first()
inverter_state = InverterState(power=power, exported=exported)
self.__store.set(inverter_state)
def can_execute_same_program_again_before_consuming_output_of_first_execution(
):
program = jq.compile(".[]+1")
first = iter(program.input([1, 2, 3]))
assert_equal(2, next(first))
second = iter(program.input([11, 12, 13]))
assert_equal(12, next(second))
assert_equal(3, next(first))
assert_equal(4, next(first))
assert_equal(13, next(second))
assert_equal(14, next(second))
def jq(self, table_name='_default', query_expression=''):
"""Find a value in a table using jq: https://stedolan.github.io/jq/
Args:
table_name (str, optional): The name of the table to query. Defaults to '_default'.
query_expression (str, optional): jq query expression. Defaults to ''.
Returns:
list: List of matched values.
"""
res = jq.compile(query_expression).input(self.table(table_name)).all()
return res
def make_nmdc_example_database(
gold_study_file='output/nmdc_etl/gold_study.json',
gold_omics_processing_file='output/nmdc_etl/gold_omics_processing.json',
gold_biosample_file='output/nmdc_etl/gold_biosample.json',
jgi_fastq_data_object_file='output/nmdc_etl/jgi_fastq_data_objects.json',
output_file='output/nmdc_example-database.json'):
## load json files
biosample_json = get_json(gold_biosample_file)
projects_json = get_json(gold_omics_processing_file)
study_json = get_json(gold_study_file)
data_objects_json = get_json(jgi_fastq_data_object_file)
## get a list of distinct omics processing study ids, and choose the first 3 studies
study_ids = set(jq.compile('.[] | .part_of[]').input(
projects_json).all()) # all returns a list
study_ids = list(study_ids)[0:3]
# study_ids =
## build a test set of studies from the study ids
study_test = \
jq.compile('.[] | select( .id == (' + ', '.join('"{0}"'.format(id) for id in study_ids) + '))')\
.input(study_json).all() # all() returns a list
## build a test set of projects from the study ids
## note: the jq query only selects first omics found for a given study id
projects_test = []
for id in study_ids:
j = jq.compile(f'[.[] | select( .part_of[]? | . == "{id}")][0]'
).input(projects_json).all()
projects_test.append(*j)
## get list of unique biossample ids from omics processing and build biosample test set
biosample_ids = jq.compile('.[] | .has_input[]?').input(
projects_test).all() # all() returns a list
biosample_test = \
jq.compile('.[] | select( .id == (' + ', '.join('"{0}"'.format(id) for id in biosample_ids) + '))')\
.input(biosample_json).all() # all() returns a list
## get a list of data object ids and build data objects test set
data_objects_ids = jq.compile('.[] | .has_output[]?').input(
projects_test).all() # all() returns a list
data_objects_test = \
jq.compile('.[] | select( .id == (' + ', '.join('"{0}"'.format(id) for id in data_objects_ids) + '))')\
.input(data_objects_json).all() # all() returns a list
## compile into database object
database = \
{
"study_set": [*study_test],
"omics_processing_set": [*projects_test],
"biosample_set": [*biosample_test],
"data_object_set": [*data_objects_test]
}
save_json(database, output_file)
def cmd_jq(self, items: Iterable, args: str) -> Iterable:
"""Usage: | jq <jq filter> |
Run jq JSON processor against the input which must
either be Cloudkeeper resources or a JSON string.
"""
compiled_jq = jq.compile(args)
for item in items:
if isinstance(item, BaseResource):
item = fmt_json(resource2dict(item, True, self.graph))
elif not isinstance(item, str):
continue
yield from compiled_jq.input(text=item).all()
def unmarshal(response, jq_expression):
"""
Unmarshal metric value from metric response
"""
try:
# in general, jq execution could yield multiple values
# we will use the first value
num = jq.compile(jq_expression).input(response).first()
# if that value is not a number, there is an error
if isinstance(num, numbers.Number) and not np.isnan(num):
return num, None
return None, ValueError("Metrics response did not yield a number")
except Exception as err:
return None, err
