def test_multinode_description(self): self.assertIsNotNone(self.client_job) allow_missing_path(self.client_job.validate, self, "qemu-system-x86_64") # check that the description can be re-loaded as valid YAML for action in self.client_job.pipeline.actions: data = action.explode() data_str = yaml.dump(data) yaml_load(data_str) # nosec not suitable for safe_load
def test_internal_v1_jobs_get(client, mocker, settings): # Create objects objs = create_objects(Worker.objects.create(hostname="worker-01")) (j1, j2, j3, j4, j5, j6) = objs["jobs"] # Test errors ret = client.get(reverse("lava.scheduler.internal.v1.jobs", args=["12345"])) assert ret.status_code == 404 ret = client.get(reverse("lava.scheduler.internal.v1.jobs", args=[j1.id])) assert ret.status_code == 400 assert ret.json()["error"] == "Missing 'token'" ret = client.get(reverse("lava.scheduler.internal.v1.jobs", args=[j1.id]), HTTP_LAVA_TOKEN="") assert ret.status_code == 400 assert ret.json()["error"] == "Invalid 'token'" ret = client.get( reverse("lava.scheduler.internal.v1.jobs", args=[j1.id]), HTTP_LAVA_TOKEN=j1.token, ) assert ret.status_code == 200 assert list(ret.json().keys()) == [ "definition", "device", "dispatcher", "env", "env-dut", ] print(ret.json()) assert yaml_load(ret.json()["definition"]) == { "compatibility": 0, "device_type": "qemu", } assert "hostname: qemu05" not in ret.json()["device"] assert "available_architectures:" in ret.json()["device"] ret = client.get( reverse("lava.scheduler.internal.v1.jobs", args=[j6.id]), HTTP_LAVA_TOKEN=j6.token, ) assert ret.status_code == 200 assert list(ret.json().keys()) == [ "definition", "device", "dispatcher", "env", "env-dut", ] assert yaml_load(ret.json()["definition"]) == { "compatibility": 0, "connection": "ssh", "host_role": "hello", } assert "hostname: qemu05" in ret.json()["device"] assert "available_architectures:" not in ret.json()["device"]
def write(self, job, line, output=None, idx=None): line = yaml_load(line)[0] doc_ref = (self.db.collection(self.root_collection).document( "%02d-%02d-%02d" % (job.submit_time.year, job.submit_time.month, job.submit_time.day)).collection(str( job.id)).document(line["dt"])) doc_ref.set({"lvl": line["lvl"], "msg": line["msg"]})
def create_metadata_store(results, job): """ Uses the OrderedDict import to correctly handle the yaml.load """ if "extra" not in results: return None level = results.get("level") if level is None: return None logger = logging.getLogger("lava-master") stub = "%s-%s-%s.yaml" % (results["definition"], results["case"], level) meta_filename = os.path.join(job.output_dir, "metadata", stub) os.makedirs(os.path.dirname(meta_filename), mode=0o755, exist_ok=True) if os.path.exists(meta_filename): with open(meta_filename, "r") as existing_store: data = yaml_load(existing_store) data.update(results["extra"]) else: data = results["extra"] try: with open(meta_filename, "w") as extra_store: yaml.dump(data, extra_store) except OSError as exc: # LAVA-847 msg = "[%d] Unable to create metadata store: %s" % (job.id, exc) logger.error(msg) append_failure_comment(job, msg) return None return meta_filename
def test_devicetype_get_health_check(self, monkeypatch, tmpdir): real_open = open (tmpdir / "qemu.yaml").write_text("hello", encoding="utf-8") def monkey_open(path, *args): if path == os.path.join(settings.HEALTH_CHECKS_PATH, "qemu.yaml"): return real_open(str(tmpdir / "qemu.yaml"), *args) if path == os.path.join(settings.HEALTH_CHECKS_PATH, "docker.yaml"): raise FileNotFoundError() return real_open(path, *args) monkeypatch.setitem(__builtins__, "open", monkey_open) # 1. normal case qemu_device_type1 = DeviceType.objects.create(name="qemu") data = self.hit( self.userclient, reverse("api-root", args=[self.version]) + "devicetypes/%s/health_check/" % qemu_device_type1.name, ) data = yaml_load(data) assert data == str("hello") # nosec # 2. Can't read the health-check docker_device_type1 = DeviceType.objects.create(name="docker") response = self.userclient.get( reverse("api-root", args=[self.version]) + "devicetypes/%s/health_check/" % docker_device_type1.name) assert response.status_code == 400 # nosec
def write(self, job, line, output=None, idx=None): line = yaml_load(line)[0] dt = datetime.datetime.strptime(line["dt"], "%Y-%m-%dT%H:%M:%S.%f") line.update({"job_id": job.id, "dt": int(dt.timestamp() * 1000)}) data = simplejson.dumps(line) requests.post("%s_doc/" % self.api_url, data=data, headers=self.headers)
def test_devices_get_dictionary(self, monkeypatch, tmpdir): # invalid context response = self.userclient.get( reverse("api-root", args=[self.version]) + "devices/public01/dictionary/?context={{") assert response.status_code == 400 # nosec # no device dict monkeypatch.setattr(Device, "load_configuration", (lambda self, job_ctx, output_format: None)) response = self.userclient.get( reverse("api-root", args=[self.version]) + "devices/public01/dictionary/") assert response.status_code == 400 # nosec # success monkeypatch.setattr( Device, "load_configuration", (lambda self, job_ctx, output_format: "device: dict"), ) data = self.hit( self.userclient, reverse("api-root", args=[self.version]) + "devices/public01/dictionary/", ) data = yaml_load(data) assert data == {"device": "dict"} # nosec
def test_internal_v1_jobs_test_auth_token(client, setup, mocker): user = User.objects.get(username="******") job01 = TestJob.objects.get(description="test job 01") write_text = mocker.Mock() mocker.patch("pathlib.Path.write_text", write_text) ret = client.get( reverse("lava.scheduler.internal.v1.jobs", args=[job01.id]), HTTP_LAVA_TOKEN=job01.token, ) assert ret.status_code == 200 job_def = yaml_load(ret.json()["definition"]) # Token not in db. assert job_def["actions"][0]["deploy"]["image"]["headers"]["PRIVATE"] == "token" # Token in db. RemoteArtifactsAuth.objects.create(name="token", token="tokenvalue", user=user) ret = client.get( reverse("lava.scheduler.internal.v1.jobs", args=[job01.id]), HTTP_LAVA_TOKEN=job01.token, ) assert ret.status_code == 200 job_def = yaml_load(ret.json()["definition"]) assert ( job_def["actions"][0]["deploy"]["image"]["headers"]["PRIVATE"] == "tokenvalue" ) # No headers present. job02 = TestJob.objects.get(description="test job 02") ret = client.get( reverse("lava.scheduler.internal.v1.jobs", args=[job02.id]), HTTP_LAVA_TOKEN=job02.token, ) assert ret.status_code == 200 # Int value in deploy action dict job01.definition = INT_VALUE_JOB_DEFINITION job01.save() ret = client.get( reverse("lava.scheduler.internal.v1.jobs", args=[job01.id]), HTTP_LAVA_TOKEN=job01.token, ) assert ret.status_code == 200 job_def = yaml_load(ret.json()["definition"])
def test_simple_mapping(tmpdir): device_info = {"foo": "bar"} add_device_container_mapping("1", device_info, "mycontainer") data = yaml_load(open(tmpdir / "1" / "usbmap.yaml")) assert data["device_info"] == device_info assert data["container"] == "mycontainer" assert data["container_type"] == "lxc"
def test_testjob_yaml(self): data = self.hit( self.userclient, reverse("api-root", args=[self.version]) + "jobs/%s/yaml/" % self.public_testjob1.id, ) data = yaml_load(data) assert data[0]["job"] == str( self.public_testjob1.id) # nosec - unit test support
def write(self, job, line, output=None, idx=None): line = yaml_load(line)[0] self.db.logs.insert_one({ "job_id": job.id, "dt": line["dt"], "lvl": line["lvl"], "msg": line["msg"] })
def load_mapping_data(filename): try: with open(filename) as f: data = yaml_load(f) or [] if isinstance(data, dict): data = [data] return data except FileNotFoundError: return []
def test_decimal_yaml_dump(self): job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(), self.user) test_dict = { "definition": "unit-test", "case": "unit-test", "measurement": decimal.Decimal(1234.5), "result": "pass", } test_case = map_scanned_results(test_dict, job, {}, None) self.assertEqual(yaml_load(test_case.metadata)["measurement"], "1234.5")
def test_metastore(self): field = TestCase._meta.get_field("metadata") level = "1.3.5.1" # artificially inflate results to represent a set of kernel messages results = { "definition": "lava", "case": "unit-test", "level": level, # list of numbers, generates a much longer YAML string than just the count "extra": list(range(int(field.max_length / 2))), "result": "pass", } stub = "%s-%s-%s.yaml" % (results["definition"], results["case"], level) job = TestJob.from_yaml_and_user(self.factory.make_job_yaml(), self.user) meta_filename = os.path.join(job.output_dir, "metadata", stub) filename = "%s/job-%s/pipeline/%s/%s-%s.yaml" % ( job.output_dir, job.id, level.split(".")[0], level, results["definition"], ) mkdir(os.path.dirname(filename)) if os.path.exists(meta_filename): # isolate from other unit tests os.unlink(meta_filename) self.assertEqual(meta_filename, create_metadata_store(results, job)) ret = map_scanned_results(results, job, {}, meta_filename) self.assertIsNotNone(ret) ret.save() self.assertEqual(TestCase.objects.filter(name="unit-test").count(), 1) test_data = yaml_load( TestCase.objects.filter(name="unit-test")[0].metadata) self.assertEqual(test_data["extra"], meta_filename) self.assertTrue(os.path.exists(meta_filename)) with open(test_data["extra"], "r") as extra_file: data = yaml_load(extra_file) self.assertIsNotNone(data) os.unlink(meta_filename) shutil.rmtree(job.output_dir)
def export_testcase(testcase, with_buglinks=False): """ Returns string versions of selected elements of a TestCase Unicode causes issues with CSV and can complicate YAML parsing with non-python parsers. :param testcase: list of TestCase objects :return: Dictionary containing relevant information formatted for export """ metadata = dict( testcase.action_metadata) if testcase.action_metadata else {} extra_source = [] extra_data = metadata.get("extra") if isinstance(extra_data, str) and os.path.exists(extra_data): with open(metadata["extra"], "r") as extra_file: # TODO: this can fail! items = yaml_load(extra_file) # hide the !!python OrderedDict prefix from the output. for key, value in items.items(): extra_source.append({key: value}) metadata["extra"] = extra_source casedict = { "name": str(testcase.name), "job": str(testcase.suite.job_id), "suite": str(testcase.suite.name), "result": str(testcase.result_code), "measurement": str(testcase.measurement), "unit": str(testcase.units), "level": metadata.get("level", ""), "url": str(testcase.get_absolute_url()), "id": str(testcase.id), "logged": str(testcase.logged), "log_start_line": str(testcase.start_log_line) if testcase.start_log_line else "", "log_end_line": str(testcase.end_log_line) if testcase.end_log_line else "", "metadata": metadata, } if with_buglinks: casedict["buglinks"] = [ str(url) for url in testcase.buglinks.values_list("url", flat=True) ] return casedict
def test_testjob_suite_yaml(self): data = self.hit( self.userclient, reverse("api-root", args=[self.version]) + "jobs/%s/suites/%s/yaml/" % (self.public_testjob1.id, self.public_testjob1.testsuite_set.first().id), ) data = yaml_load(data) assert ( data[0]["suite"] == self.public_testjob1.testsuite_set.first().name ) # nosec - unit test support
def check(logger, lvl, lvlno, msg=None, mock_calls=1): assert len(logger._log.mock_calls) == mock_calls if mock_calls == 0: return assert logger._log.mock_calls[0][1][0] == lvlno data = yaml_load(logger._log.mock_calls[0][1][1]) assert list(data.keys()) == ["dt", "lvl", "msg"] assert data["lvl"] == lvl if msg is None: assert data["msg"] == f"an {lvl}" else: assert data["msg"] == msg
def test_mongo_logs(mocker): mocker.patch("pymongo.database.Database.command") mocker.patch("pymongo.collection.Collection.create_index") logs_mongo = LogsMongo() job = mocker.Mock() job.id = 1 insert_one = mocker.MagicMock() find = mocker.MagicMock() find_ret_val = [ { "dt": "2020-03-25T19:44:36.209548", "lvl": "info", "msg": "first message" }, { "dt": "2020-03-26T19:44:36.209548", "lvl": "info", "msg": "second message" }, ] find.return_value = find_ret_val mocker.patch("pymongo.collection.Collection.find", find) mocker.patch("pymongo.collection.Collection.insert_one", insert_one) logs_mongo.write( job, '- {"dt": "2020-03-25T19:44:36.209548", "lvl": "info", "msg": "lava-dispatcher, installed at version: 2020.02"}', ) insert_one.assert_called_with({ "job_id": 1, "dt": "2020-03-25T19:44:36.209548", "lvl": "info", "msg": "lava-dispatcher, installed at version: 2020.02", }) # nosec result = yaml_load(logs_mongo.read(job)) assert len(result) == 2 # nosec assert result == find_ret_val # nosec # size of find_ret_val in bytes assert logs_mongo.size(job) == 137 # nosec assert logs_mongo.open(job).read() == yaml_dump(find_ret_val).encode( "utf-8")
def _get_docs(self, job, start=0, end=None): if not end: end = self.MAX_RESULTS limit = end - start if limit < 0: return [] params = { "query": { "match": { "job_id": job.id } }, "from": start, "size": limit, "sort": [{ "dt": { "order": "asc" } }], } response = requests.get( "%s_search/" % self.api_url, data=simplejson.dumps(params), headers=self.headers, ) response = simplejson.loads(response.text) if not "hits" in response: return [] result = [] for res in response["hits"]["hits"]: doc = res["_source"] doc.update({ "dt": datetime.datetime.fromtimestamp(doc["dt"] / 1000.0).isoformat() }) if doc["lvl"] == "results": doc.update({"msg": yaml_load(doc["msg"])}) result.append(doc) return result
def test_wait_for_board_id_is_optional(factory): action = DockerTestAction() action.job = Job("1234", {}, None) rendered, _ = factory.create_device("hi6220-hikey-r2-01.jinja2") action.job.device = NewDevice(yaml_load(rendered)) action.job.timeout = Timeout("blah") action.level = 1 action.populate( { "namespace": "common", "docker": {"image": "foobar", "wait": {"device": False}}, } ) assert not any( [a for a in action.pipeline.actions if a.name == "wait-device-boardid"] ) docker_test_shell = action.pipeline.actions[-2] assert not docker_test_shell.wait_for_device
def test_workers_get_config(self, monkeypatch, tmpdir): (tmpdir / self.worker1.hostname).mkdir() (tmpdir / self.worker1.hostname / "dispatcher.yaml").write_text( "hello world", encoding="utf-8") class MyPath(pathlib.PosixPath): def __new__(cls, path, *args, **kwargs): if (path == "%s/worker1/dispatcher.yaml" % settings.DISPATCHER_CONFIG_PATH): return super().__new__( cls, str(tmpdir / "worker1" / "dispatcher.yaml"), *args, **kwargs) elif path == "%s/worker1.yaml" % settings.GLOBAL_SETTINGS_PATH: return super().__new__(cls, str(tmpdir / "worker1.yaml"), *args, **kwargs) else: assert 0 # nosec monkeypatch.setattr(pathlib, "Path", MyPath) data = self.hit( self.userclient, reverse("api-root", args=[self.version]) + "workers/%s/config/" % self.worker1.hostname, ) data = yaml_load(data) assert data == str("hello world") # nosec # worker does not exists response = self.userclient.get( reverse("api-root", args=[self.version]) + "workers/invalid_hostname/config/") assert response.status_code == 404 # nosec # no configuration file (tmpdir / self.worker1.hostname / "dispatcher.yaml").remove() (tmpdir / self.worker1.hostname).remove() response = self.userclient.get( reverse("api-root", args=[self.version]) + "workers/%s/config/" % self.worker1.hostname) assert response.status_code == 400 # nosec
def parse_job_description(job): filename = os.path.join(job.output_dir, "description.yaml") logger = logging.getLogger("lava-master") try: with open(filename, "r") as f_describe: description = f_describe.read() pipeline = yaml_load(description) except (OSError, yaml.YAMLError): logger.error("'Unable to open and parse '%s'", filename) return if not map_metadata(description, job): logger.warning("[%d] unable to map metadata", job.id) # add the compatibility result from the master to the definition for comparison on the slave. try: compat = int(pipeline["compatibility"]) except (TypeError, ValueError): compat = pipeline["compatibility"] if pipeline is not None else None logger.error("[%d] Unable to parse job compatibility: %s", job.id, compat) compat = 0 job.pipeline_compatibility = compat job.save(update_fields=["pipeline_compatibility"])
def test_mapping_for_new_container_overrides_previous_mapping(tmpdir): add_device_container_mapping( "1", { "serial_number": "1234567890", "vendor_id": None, "product_id": None, "fs_label": None, }, "mycontainer1", ) add_device_container_mapping( "1", { "serial_number": "1234567890", "vendor_id": None, "product_id": None, "fs_label": None, }, "mycontainer2", ) data = yaml_load(open(tmpdir / "1" / "usbmap.yaml")) assert len(data) == 1 assert data[0]["container"] == "mycontainer2"
def make_job_data(self, actions=None, **kw): sample_job_file = os.path.join(os.path.dirname(__file__), "qemu.yaml") with open(sample_job_file, "r") as test_support: data = yaml_load(test_support) data.update(kw) return data
def logging_socket(self): msg = self.log_socket.recv_multipart() try: (job_id, message) = (u(m) for m in msg) except UnicodeDecodeError: self.logger.error("[POLL] Invalid log message: can't be decoded") return except ValueError: # do not let a bad message stop the master. self.logger.error( "[POLL] failed to parse log message, skipping: %s", msg) return try: scanned = yaml_load(message) except yaml.YAMLError: self.logger.error("[%s] data are not valid YAML, dropping", job_id) return # Look for "results" level try: message_lvl = scanned["lvl"] message_msg = scanned["msg"] except TypeError: self.logger.error("[%s] not a dictionary, dropping", job_id) return except KeyError: self.logger.error( '[%s] invalid log line, missing "lvl" or "msg" keys: %s', job_id, message, ) return # Find the handler (if available) if job_id not in self.jobs: # Query the database for the job try: job = TestJob.objects.get(id=job_id) except TestJob.DoesNotExist: self.logger.error("[%s] unknown job id", job_id) return self.logger.info("[%s] receiving logs from a new job", job_id) # Create the sub directories (if needed) mkdir(job.output_dir) self.jobs[job_id] = JobHandler(job) # For 'event', send an event and log as 'debug' if message_lvl == "event": self.logger.debug("[%s] event: %s", job_id, message_msg) send_event(".event", "lavaserver", { "message": message_msg, "job": job_id }) message_lvl = "debug" # For 'marker', save in the database and log as 'debug' elif message_lvl == "marker": # TODO: save on the file system in case of lava-logs restart m_type = message_msg.get("type") case = message_msg.get("case") if m_type is None or case is None: self.logger.error("[%s] invalid marker: %s", job_id, message_msg) return # This is in fact the previous line line = self.jobs[job_id].line_count() - 1 self.jobs[job_id].markers.setdefault(case, {})[m_type] = line return # Mark the file handler as used self.jobs[job_id].last_usage = time.time() # The format is a list of dictionaries self.jobs[job_id].write("- %s" % message) if message_lvl == "results": try: job = TestJob.objects.get(pk=job_id) except TestJob.DoesNotExist: self.logger.error("[%s] unknown job id", job_id) return meta_filename = create_metadata_store(message_msg, job) new_test_case = map_scanned_results( results=message_msg, job=job, markers=self.jobs[job_id].markers, meta_filename=meta_filename, ) if new_test_case is None: self.logger.warning("[%s] unable to map scanned results: %s", job_id, message) else: self.test_cases.append(new_test_case) # Look for lava.job result if (message_msg.get("definition") == "lava" and message_msg.get("case") == "job"): # Flush cached test cases self.flush_test_cases() if message_msg.get("result") == "pass": health = TestJob.HEALTH_COMPLETE health_msg = "Complete" else: health = TestJob.HEALTH_INCOMPLETE health_msg = "Incomplete" self.logger.info("[%s] job status: %s", job_id, health_msg) infrastructure_error = message_msg.get("error_type") in [ "Bug", "Configuration", "Infrastructure", ] if infrastructure_error: self.logger.info("[%s] Infrastructure error", job_id) # Update status. with transaction.atomic(): # TODO: find a way to lock actual_device job = TestJob.objects.select_for_update().get(id=job_id) job.go_state_finished(health, infrastructure_error) job.save()
job = get_restricted_job(request.user, pk=job, request=request) if not pk: test_suite = case.suite else: test_suite = get_object_or_404(TestSuite, name=pk, job=job) if test_sets: # No test case was found. test_sets = test_sets.filter(suite=test_suite) test_cases = TestCase.objects.none() else: test_cases = TestCase.objects.filter(name=case.name, suite=test_suite) extra_source = {} logger = logging.getLogger("lava-master") for extra_case in test_cases: try: f_metadata = yaml_load(extra_case.metadata) if not f_metadata: continue except TypeError: logger.info("Unable to load extra case metadata for %s", extra_case) continue extra_data = f_metadata.get("extra") try: if extra_data and os.path.exists(extra_data): with open(f_metadata["extra"], "r") as extra_file: items = yaml_load(extra_file) # hide the !!python OrderedDict prefix from the output. for key, value in items.items(): extra_source.setdefault(extra_case.id, "") extra_source[extra_case.id] += "%s: %s\n" % (key, value)
def map_metadata(description, job): """ Generate metadata from the combination of the pipeline definition file (after any parsing for protocols) and the pipeline description into static metadata (TestData) related to this specific job The description itself remains outside the database - it will need to be made available as a download link. :param description: the pipeline description output :param job: the TestJob to associate :return: True on success, False on error """ logger = logging.getLogger("lava-master") try: submission_data = yaml_safe_load(job.definition) description_data = yaml_load(description) except yaml.YAMLError as exc: logger.exception("[%s] %s", job.id, exc) return False try: testdata, created = TestData.objects.get_or_create(testjob=job) except MultipleObjectsReturned: # only happens for small number of jobs affected by original bug. logger.info("[%s] skipping alteration of duplicated TestData", job.id) return False if not created: # prevent updates of existing TestData logger.debug("[%s] skipping alteration of existing TestData", job.id) return False # get job-action metadata if description is None: logger.warning("[%s] skipping empty description", job.id) return False if not description_data: logger.warning("[%s] skipping invalid description data", job.id) return False if "job" not in description_data: logger.warning("[%s] skipping description without a job.", job.id) return False action_values = _get_action_metadata(description_data["job"]["actions"]) for key, value in action_values.items(): if not key or not value: logger.warning("[%s] Missing element in job. %s: %s", job.id, key, value) continue testdata.attributes.create(name=key, value=value) # get common job metadata job_metadata = _get_job_metadata(job) for key, value in job_metadata.items(): testdata.attributes.create(name=key, value=value) # get metadata from device device_values = {} device_values["target.device_type"] = job.requested_device_type for key, value in device_values.items(): if not key or not value: logger.warning("[%s] Missing element in device. %s: %s", job.id, key, value) continue testdata.attributes.create(name=key, value=value) # Add metadata from job submission data. if "metadata" in submission_data: for key in submission_data["metadata"]: value = submission_data["metadata"][key] if not key or not value: logger.warning("[%s] Missing element in job. %s: %s", job.id, key, value) continue testdata.attributes.create(name=key, value=value) walk_actions(description_data["pipeline"], testdata, submission_data) return True
def test_elasticsearch_logs(mocker, logs_elasticsearch): job = mocker.Mock() job.id = 1 post = mocker.MagicMock() get = mocker.MagicMock() get_ret_val = mocker.Mock() # Test with empty object first. get_ret_val.text = "{}" get.return_value = get_ret_val mocker.patch("requests.get", get) result = logs_elasticsearch.read(job) assert result == "" # Normal test. get_ret_val.text = '{"hits":{"hits":[{"_source":{"dt": 1585165476209, "lvl": "info", "msg": "first message"}}, {"_source":{"dt": 1585165476210, "lvl": "info", "msg": "second message"}}]}}' get.return_value = get_ret_val mocker.patch("requests.get", get) mocker.patch("requests.post", post) line = '- {"dt": "2020-03-25T19:44:36.209", "lvl": "info", "msg": "lava-dispatcher, installed at version: 2020.02"}' logs_elasticsearch.write(job, line) post.assert_called_with( "%s%s/_doc/" % (settings.ELASTICSEARCH_URI, settings.ELASTICSEARCH_INDEX), data= '{"dt": 1585165476209, "lvl": "info", "msg": "lava-dispatcher, installed at version: 2020.02", "job_id": 1}', headers={"Content-type": "application/json"}, ) # nosec result = yaml_load(logs_elasticsearch.read(job)) assert len(result) == 2 # nosec assert result == [ { "dt": "2020-03-25T19:44:36.209000", "lvl": "info", "msg": "first message" }, { "dt": "2020-03-25T19:44:36.210000", "lvl": "info", "msg": "second message" }, ] # nosec # size of get_ret_val in bytes assert logs_elasticsearch.size(job) == 137 # nosec assert (logs_elasticsearch.open(job).read() == yaml_dump([ { "dt": "2020-03-25T19:44:36.209000", "lvl": "info", "msg": "first message", }, { "dt": "2020-03-25T19:44:36.210000", "lvl": "info", "msg": "second message", }, ]).encode("utf-8"))
def find_mapping(options): for mapping in glob.glob(get_mapping_path("*")): data = yaml_load(open(mapping)) if match_mapping(data["device_info"], options): return data return None