def value_error_is_raised_if_program_is_invalid(): try: jq.compile("!") assert False, "Expected error" except ValueError as error: expected_error_str = "jq: error: syntax error, unexpected INVALID_CHARACTER, expecting $end (Unix shell quoting issues?) at <top-level>, line 1:\n!\njq: 1 compile error" assert_equal(str(error), expected_error_str)
def update(self, response) -> None: log.MainLogger().debug("Komponente "+self.component_config["name"]+" auslesen.") config = self.component_config["configuration"] power = float(jq.compile(config["jq_power"]).input(response).first()) if power >= 0: power = power * -1 if config["jq_counter"] == "": topic_str = "openWB/set/system/device/" + \ str(self.__device_id)+"/component/" + \ str(self.component_config["id"])+"/" _, counter = self.__sim_count.sim_count( power, topic=topic_str, data=self.simulation, prefix="pv") inverter_state = InverterState( power=power, counter=counter, currents=[0, 0, 0] ) else: counter = jq.compile(config["jq_counter"]).input(response).first() inverter_state = InverterState( power=power, counter=counter ) self.__store.set(inverter_state)
def update(self, response) -> None: config = self.component_config.configuration power = jq.compile(config.jq_power).input(response).first() if config.jq_soc != "": soc = jq.compile(config.jq_soc).input(response).first() else: soc = 0 if config.jq_imported != "" and config.jq_exported != "": imported = jq.compile(config.jq_imported).input(response).first() exported = jq.compile(config.jq_exported).input(response).first() else: topic_str = "openWB/set/system/device/" + str( self.__device_id)+"/component/"+str(self.component_config.id)+"/" imported, exported = self.__sim_count.sim_count( power, topic=topic_str, data=self.simulation, prefix="speicher" ) bat_state = BatState( power=power, soc=soc, imported=imported, exported=exported ) self.__store.set(bat_state)
def update(self, response): log.MainLogger().debug("Komponente "+self.component_config["name"]+" auslesen.") config = self.component_config["configuration"] power = jq.compile(config["jq_power"]).input(response).first() if config["jq_imported"] == "" or config["jq_exported"] == "": topic_str = "openWB/set/system/device/{}/component/{}/".format( self.__device_id, self.component_config["id"] ) imported, exported = self.__sim_count.sim_count( power, topic=topic_str, data=self.simulation, prefix="bezug" ) else: imported = jq.compile(config["jq_imported"]).input(response).first() exported = jq.compile(config["jq_exported"]).input(response).first() counter_state = CounterState( imported=imported, exported=exported, power=power ) self.__store.set(counter_state)
def delete_comments(repo, pull_number): print('clearing comments') bot = 'github-actions[bot]' comment_url = ISSUE_COMMENTS.format(repo=repo, issue_number=pull_number) comments = request('get', comment_url) jq_user = jq.compile('.user.login') jq_comment = jq.compile('.id') for comment in comments: user = jq_user.input(comment).first() if user == bot: comment_id = jq_comment.input(comment).first() delete_comment(repo, comment_id)
def compiling_with_args_should_set_predefined_variables(): assert_equal( 123, jq.compile("$a + $b + .", args={ "a": 100, "b": 20 }).input(3).first())
async def locationsv1_get( db: DB = Depends(), locations: Locations = Depends(Locations.depends()), ): data = await locations_get(db, locations) meta = data.meta res = data.results latest_jq = jq.compile(""" .[] | { id: .id, country: .country, city: .city, location: .name, soureName: .source_name, sourceType: .sources[0].name, coordinates: .coordinates, firstUpdated: .firstUpdated, lastUpdated: .lastUpdated, parameters : [ .parameters[].parameter ], countsByMeasurement: [ .parameters[] | { parameter: .parameter, count: .count } ], count: .parameters| map(.count) | add } """) ret = latest_jq.input(res).all() return OpenAQResult(meta=meta, results=ret)
def program_preserves_floats(): program = jq.compile(".") assert_equal(float, type(program.input(text="1.1").first())) assert_equal(3.14159, program.input(text="3.14159").first()) assert_equal(-3.14159, program.input(text="-3.14159").first()) assert_equal(42E100, program.input(text="42E100").first()) assert_equal(-42E100, program.input(text="-42E100").first())
def jq_select(expression: str, obj): if has_jq: return jq.compile(expression).input(obj).all() else: return json.loads( subprocess.check_output(['jq', expression], input=json.dumps(obj).encode()))
def main(): """Execute PY_SETUP (if set) and call PY_TARGET on JSON from stdin. See https://github.com/wbadart/rmq_py_caller#readme for details. """ parser = ArgumentParser(prog="python -m rmq_py_caller", description=main.__doc__) parser.add_argument( "-v", "--verbose", action="count", default=0, help="-v to include info, -vv to include debugging", ) args = parser.parse_args() # If you're curious about the use exec/ eval here, please see the project # wiki for discussion. # pylint: disable=exec-used,eval-used exec(environ.get("PY_SETUP", ""), globals()) ctx = eval(environ["PY_TARGET"]) # Wait until now to call basicConfig in case PY_SETUP wants to do it if args.verbose: logging.basicConfig( level=[logging.WARN, logging.INFO, logging.DEBUG][args.verbose]) log = logging.getLogger(__name__) log.debug("Got PY_TARGET: %s", ctx) adapter = jq.compile(environ.get("ARG_ADAPTER", "[.]")) log.debug("Using ARG_ADAPTER: %s", adapter.program_string) main_loop(ctx, adapter, sys.stdin)
async def latest_get( db: DB = Depends(), locations: Locations = Depends(Locations.depends()), ): data = await locations_get(db, locations) meta = data.meta res = data.results if len(res) == 0: return data latest_jq = jq.compile(""" .[] | { location: .name, city: .city, country: .country, coordinates: .coordinates, measurements: [ .parameters[] | { parameter: .measurand, value: .lastValue, lastUpdated: .lastUpdated, unit: .unit } ] } """) ret = latest_jq.input(res).all() return OpenAQResult(meta=meta, results=ret)
def program_preserves_arrays(): program = jq.compile(".") assert_equal(list, type(program.input(text='[]').first())) assert_equal([], program.input(text='[]').first()) assert_equal([1], program.input(text='[1]').first()) assert_equal([1, 2], program.input(text='[1, 2]').first()) assert_equal([3.14159], program.input(text='[3.14159]').first()) assert_equal([3.14159, 95141.3], program.input(text='[3.14159, 95141.3]').first()) assert_equal([False], program.input(text='[false]').first()) assert_equal([False, True], program.input(text='[false, true]').first()) assert_equal([[]], program.input(text='[[]]').first()) assert_equal([[[]]], program.input(text='[[[]]]').first()) assert_equal([[], []], program.input(text='[[], []]').first()) assert_equal([[[], []], [[], []]], program.input(text='[[[], []], [[], []]]').first()) assert_equal([{}], program.input(text='[{}]').first()) assert_equal([{"": []}], program.input(text='[{"": []}]').first()) assert_equal([{ "1": [], "2": [] }], program.input(text='[{"1": [], "2": []}]').first())
def ps_hook(tasking: models.Tasking): """ This hook watches for the 'ps' command and writes the processes into the processes table. For Powershell Agents, the data comes back (as of 4.1) as JSON. For Python Agents, the data comes back in the typical 'ls' format. For C# Agents, no support yet. AFAIK, it is not easy to convert the shell tables into JSON, but I found this jq wizardry on StackOverflow, so that's what we'll stick with for now for the python results, even though it is imperfect. https://unix.stackexchange.com/a/243485 """ if (tasking.input.strip() not in ["ps", "tasklist"] or tasking.agent.language == "csharp"): return if tasking.agent.language == "python": output = (jq.compile( """[sub("\n$";"") | splits("\n") | sub("^ +";"") | [splits(" +")]] | .[0] as $header | .[1:] | [.[] | [. as $x | range($header | length) | {"key": $header[.], "value": $x[.]}] | from_entries]""" ).input( tasking.output.decode("utf-8").split( "\r\n ..Command execution completed.")[0]).first()) else: output = json.loads(tasking.output) existing_processes = (Session().query( models.HostProcess.process_id).filter( models.HostProcess.host_id == tasking.agent.host_id).all()) existing_processes = list(map(lambda p: p[0], existing_processes)) for process in output: process_name = process.get("CMD") or process.get("ProcessName") or "" process_id = process.get("PID") arch = process.get("Arch") user = process.get("UserName") if process_id: # and process_id.isnumeric(): if int(process_id) not in existing_processes: Session().add( models.HostProcess( host_id=tasking.agent.host_id, process_id=process_id, process_name=process_name, architecture=arch, user=user, )) elif int(process_id) in existing_processes: db_process: models.HostProcess = (Session().query( models.HostProcess).filter( and_( models.HostProcess.host_id == tasking.agent.host_id, models.HostProcess.process_id == process_id, )).first()) if not db_process.agent: db_process.architecture = arch db_process.process_name = process_name Session().commit()
def value_error_is_raised_if_input_is_not_valid_json(): program = jq.compile(".x") try: program.input(text="!!").first() assert False, "Expected error" except jq.JSONParseError as error: expected_error_str = "Invalid numeric literal at EOF at line 1, column 2" assert_equal(str(error), expected_error_str)
def value_error_is_raised_if_input_cannot_be_processed_by_program(): program = jq.compile(".x") try: program.input(1).all() assert False, "Expected error" except ValueError as error: expected_error_str = "Cannot index number with string \"x\"" assert_equal(str(error), expected_error_str)
def getAndUploadSettings(project): """ Use CircleCI API and jq to pull down a project's Settings from OnPrem and upload them to the active SaaS project. """ settings = compile('."feature_flags" | del(."builds-service") | del(."fleet")').input(getSettings(project)).text() print('Successfully got settings for project {}:\n{}'.format(project, settings)) uploadSettings(settings)
def test_resolve_employee_detail(self, user, api_client, employee): api_client.force_authenticate(user) response = api_client.get(reverse('employee-detail', args=(1, ))).json() jq_filtered_response = jq.compile( '. as $data | {name: $data.name}').input(response).first() expected = {'name': 'John'} assert response == expected or jq_filtered_response == expected
def program_preserves_ints(): program = jq.compile(".") assert_equal(int, type(program.input(text="0").first())) assert_equal(0, program.input(text="0").first()) assert_equal(1, program.input(text="1").first()) assert_equal(-1, program.input(text="-1").first()) assert_equal(12345, program.input(text="12345").first()) assert_equal(-12345, program.input(text="-12345").first())
def errors_do_not_leak_between_transformations(): program = jq.compile(".x") try: program.input(1).all() assert False, "Expected error" except ValueError as error: pass assert_equal(1, program.input({"x": 1}).first())
def __init__(self, **kwargs): self.config = kwargs['config'] self.log = get_logger(name=__class__.__name__) self.cells = defaultdict(list) self.sort_results = dict() if len(self.config.sort_queries) > 0: self.sort_queries = self.config.sort_queries else: self.sort_queries = [ jq.compile('.id') ]
def update(self, response): config = self.component_config.configuration power = jq.compile(config.jq_power).input(response).first() # ToDo: add current or power per phase if config.jq_imported == "" or config.jq_exported == "": topic_str = "openWB/set/system/device/{}/component/{}/".format( self.__device_id, self.component_config.id) imported, exported = self.__sim_count.sim_count( power, topic=topic_str, data=self.simulation, prefix="bezug") else: imported = jq.compile(config.jq_imported).input(response).first() exported = jq.compile(config.jq_exported).input(response).first() counter_state = CounterState(imported=imported, exported=exported, power=power) self.__store.set(counter_state)
def filter_copy(source, filter, size=500, use_float=False): """Permits response nested key filtering. Args: source (obj): Raw server response to filter. filter (obj): Whitelist filter(s) to apply to the source data. Must be a :py:class:`list` to use the native DSL, or a string to use a JQ script. This represents data you want to explicitly keep. size (int, optional): Array buffer size. (default: ``500``) Notes: The source data must not be pre-processed by a JSON converter. The native filtering DSL is simpler than JQ script, and more human readable. JQ scripts are executed per item returned, not against the entire JSON as a whole. JQ scripts provide significantly more features than the native DSL at the cost of performance. """ import ijson if isinstance(filter, str): import jq _jq = True use_float = True elif isinstance(filter, list): _jq = False def apply(_s, _f): _out = {} for i in _f: x = _filter(_s, i, _out) _out = x if x else _out return _out out = [None] * size idx = 0 if not _jq: # to remove ijson BytesIO warning for x in ijson.items(BytesIO(source.encode('utf-8')), 'item', use_float=use_float): if idx >= len(out): out.extend([None] * size) out[idx] = deepcopy(apply(x, filter)) idx += 1 del out[idx:] return out else: return deepcopy(jq.compile(filter).input(text=source).first())
def iterators_from_same_program_and_input_are_independent(): program_with_input = jq.compile(".[]+1").input([1, 2, 3]) first = iter(program_with_input) assert_equal(2, next(first)) second = iter(program_with_input) assert_equal(2, next(second)) assert_equal(3, next(first)) assert_equal(4, next(first)) assert_equal(3, next(second)) assert_equal(4, next(second))
def post_process_config(config: TConfig, args: ConfigPostProcessFlags) -> TConfig: """Post-processes a configuration according to the given flags.""" config_type = type(config) config = config.to_dict() for jq_transform in args.jq_transform: config = jq.compile(jq_transform).input(config).first() config = config_type.from_dict(config) str_template_subs = configuration.parse_template_mapping(args.string_def) config = configuration.replace_templates(config, str_template_subs) return config
def update(self, response) -> None: config = self.component_config.configuration power = float(jq.compile(config.jq_power).input(response).first()) if power >= 0: power = power * -1 if config.jq_exported == "": topic_str = "openWB/set/system/device/" + \ str(self.__device_id)+"/component/" + \ str(self.component_config.id)+"/" _, exported = self.__sim_count.sim_count( power, topic=topic_str, data=self.simulation, prefix="pv%s" % ("" if self.component_config.id == 1 else "2")) else: exported = jq.compile(config.jq_exported).input(response).first() inverter_state = InverterState(power=power, exported=exported) self.__store.set(inverter_state)
def can_execute_same_program_again_before_consuming_output_of_first_execution( ): program = jq.compile(".[]+1") first = iter(program.input([1, 2, 3])) assert_equal(2, next(first)) second = iter(program.input([11, 12, 13])) assert_equal(12, next(second)) assert_equal(3, next(first)) assert_equal(4, next(first)) assert_equal(13, next(second)) assert_equal(14, next(second))
def jq(self, table_name='_default', query_expression=''): """Find a value in a table using jq: https://stedolan.github.io/jq/ Args: table_name (str, optional): The name of the table to query. Defaults to '_default'. query_expression (str, optional): jq query expression. Defaults to ''. Returns: list: List of matched values. """ res = jq.compile(query_expression).input(self.table(table_name)).all() return res
def make_nmdc_example_database( gold_study_file='output/nmdc_etl/gold_study.json', gold_omics_processing_file='output/nmdc_etl/gold_omics_processing.json', gold_biosample_file='output/nmdc_etl/gold_biosample.json', jgi_fastq_data_object_file='output/nmdc_etl/jgi_fastq_data_objects.json', output_file='output/nmdc_example-database.json'): ## load json files biosample_json = get_json(gold_biosample_file) projects_json = get_json(gold_omics_processing_file) study_json = get_json(gold_study_file) data_objects_json = get_json(jgi_fastq_data_object_file) ## get a list of distinct omics processing study ids, and choose the first 3 studies study_ids = set(jq.compile('.[] | .part_of[]').input( projects_json).all()) # all returns a list study_ids = list(study_ids)[0:3] # study_ids = ## build a test set of studies from the study ids study_test = \ jq.compile('.[] | select( .id == (' + ', '.join('"{0}"'.format(id) for id in study_ids) + '))')\ .input(study_json).all() # all() returns a list ## build a test set of projects from the study ids ## note: the jq query only selects first omics found for a given study id projects_test = [] for id in study_ids: j = jq.compile(f'[.[] | select( .part_of[]? | . == "{id}")][0]' ).input(projects_json).all() projects_test.append(*j) ## get list of unique biossample ids from omics processing and build biosample test set biosample_ids = jq.compile('.[] | .has_input[]?').input( projects_test).all() # all() returns a list biosample_test = \ jq.compile('.[] | select( .id == (' + ', '.join('"{0}"'.format(id) for id in biosample_ids) + '))')\ .input(biosample_json).all() # all() returns a list ## get a list of data object ids and build data objects test set data_objects_ids = jq.compile('.[] | .has_output[]?').input( projects_test).all() # all() returns a list data_objects_test = \ jq.compile('.[] | select( .id == (' + ', '.join('"{0}"'.format(id) for id in data_objects_ids) + '))')\ .input(data_objects_json).all() # all() returns a list ## compile into database object database = \ { "study_set": [*study_test], "omics_processing_set": [*projects_test], "biosample_set": [*biosample_test], "data_object_set": [*data_objects_test] } save_json(database, output_file)
def cmd_jq(self, items: Iterable, args: str) -> Iterable: """Usage: | jq <jq filter> | Run jq JSON processor against the input which must either be Cloudkeeper resources or a JSON string. """ compiled_jq = jq.compile(args) for item in items: if isinstance(item, BaseResource): item = fmt_json(resource2dict(item, True, self.graph)) elif not isinstance(item, str): continue yield from compiled_jq.input(text=item).all()
def unmarshal(response, jq_expression): """ Unmarshal metric value from metric response """ try: # in general, jq execution could yield multiple values # we will use the first value num = jq.compile(jq_expression).input(response).first() # if that value is not a number, there is an error if isinstance(num, numbers.Number) and not np.isnan(num): return num, None return None, ValueError("Metrics response did not yield a number") except Exception as err: return None, err