def test_construct_duplicate(self, fake_mtime, db_session):
""" Test dataset constructor
"""
template = Template(
name="run",
idxname="run-data",
template_name="tname",
file="run.json",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
template.add()
with pytest.raises(TemplateDuplicate) as e:
template1 = Template(
name="run",
idxname="run-data",
template_name="tname",
file="run.json",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
template1.add()
assert str(e).find("run") != -1
def get(self, index_name: AnyStr) -> Dict[AnyStr, Any]:
"""
Return mapping properties of the document specified by the index name in the URI.
For example, user can get the run document properties by making a GET request on
index/mappings/run. Similarly other index documents can be fetched by making a GET
request on appropriate index names.
We fetch the mapping by querying the template database. If the template is not found
in the database NOT_FOUND error will be raised.
"""
try:
template = Template.find(index_name)
mappings = template.mappings
result = {}
for property in mappings["properties"]:
if "properties" in mappings["properties"][property]:
result[property] = list(
mappings["properties"][property]["properties"].keys())
# construct response object
return jsonify(result)
except TemplateNotFound:
self.logger.exception(
"Document template {} not found in the database.", index_name)
abort(HTTPStatus.NOT_FOUND, message="Mapping not found")
def test_construct_missing(self, fake_mtime, db_session):
""" Test dataset constructor when non-nullable columns are omitted;
the constuctor works, but SQL will throw an IntegrityError when we
try to commit to the DB.
"""
with pytest.raises(TemplateMissingParameter) as e:
template = Template(
name="run",
file="map.json",
template_name="tname",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
version=5,
)
template.add()
assert str(e).find("run") != -1
def test_construct(self, fake_mtime, db_session):
""" Test dataset constructor
"""
template = Template(
name="run",
idxname="run-data",
template_name="tname",
file="run.json",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
template.add()
assert template.name == "run"
assert template.mtime == datetime.datetime(2021, 1, 29, 0, 0, 0)
assert "run: drb.v1.run.{year}-{month}" == str(template)
def test_find_exists(self, fake_mtime, db_session):
""" Test that we can find a template
"""
template1 = Template(
name="run",
idxname="run-data",
template_name="run",
file="run-toc.json",
template_pattern="drb.v2.run-toc.*",
index_template="drb.v2.run-toc.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
template1.add()
template2 = Template.find(name="run")
assert template2.name == template1.name
assert template2.id is template1.id
def test_construct_fileless(self, fake_mtime, db_session):
""" Test dataset constructor without a file column
"""
with pytest.raises(TemplateFileMissing) as e:
Template(
name="run",
idxname="run-data",
template_name="tname",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
assert str(e).find("run") != -1
def test_update(self, fake_mtime, db_session):
""" Test template update
"""
template = Template(
name="run",
file="run.json",
idxname="run-data",
template_name="tname",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
template.add()
template.mappings = {"properties": "something"}
template.update()
def test_update_missing(self, fake_mtime, db_session):
""" Test template update
"""
template = Template(
name="run",
file="run.json",
idxname="run-data",
template_name="tname",
template_pattern="drb.v1.run.*",
index_template="drb.v1.run.{year}-{month}",
settings={"none": False},
mappings={"properties": None},
version=5,
)
template.add()
template.idxname = None
with pytest.raises(TemplateMissingParameter) as e:
template.update()
assert str(e).find("run") != -1
assert str(e).find("idxname") != -1
def _gen_month_range(self, index: str, start: datetime,
end: datetime) -> str:
"""
Construct a comma-separated list of index names qualified by year and
month suitable for use in the Elasticsearch /_search query URI.
The month is incremented by 1 from "start" to "end"; for example,
_gen_month_range('run', '2020-08', '2020-10') might result
in
'drb.v4.run.2020-08,drb.v4.run.2020-09,drb.v4.run.2020-10,'
Args:
index: The desired monthly index root (e.g., 'run')
start: The start time
end: The end time
Returns:
A comma-separated list of month-qualified index names
"""
template = Template.find(index)
indices = ""
first_month = start.replace(day=1)
last_month = end + relativedelta(day=31)
for m in rrule.rrule(rrule.MONTHLY,
dtstart=first_month,
until=last_month):
indices += (template.index_template.format(
prefix=self.prefix,
version=template.version,
idxname=template.idxname,
year=f"{m.year:04}",
month=f"{m.month:02}",
day="*",
) + ",")
return indices
def fake_find(name: str) -> Template:
if name == "run":
return Template(
name="run",
idxname="run-data",
template_name="unit-test.v6.run-data",
file="run.json",
template_pattern="unit-test.v6.run-data.*",
index_template="unit-test.v6.run-data.{year}-{month}",
settings={"none": False},
mappings={
"_meta": {
"version": "6"
},
"date_detection": "false",
"properties": {
"@generated-by": {
"type": "keyword"
},
"@metadata": {
"properties": {
"controller_dir": {
"type": "keyword"
},
"file-date": {
"type": "date"
},
"file-name": {
"type": "keyword"
},
"file-size": {
"type": "long"
},
"md5": {
"type": "keyword"
},
"pbench-agent-version": {
"type": "keyword"
},
"raw_size": {
"type": "long"
},
"result-prefix": {
"type": "text"
},
"satellite": {
"type": "keyword"
},
"tar-ball-creation-timestamp": {
"type": "date"
},
"toc-prefix": {
"type": "text"
},
}
},
"@timestamp": {
"type": "date"
},
"authorization": {
"properties": {
"access": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
}
},
},
"owner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
}
},
},
}
},
"host_tools_info": {
"type": "nested",
"properties": {
"hostname": {
"type": "keyword"
},
"hostname-f": {
"type": "keyword"
},
"hostname-s": {
"type": "keyword"
},
"label": {
"type": "keyword"
},
"tools": {
"properties": {
"disk": {
"type": "text"
},
"haproxy-ocp": {
"type": "text"
},
"iostat": {
"type": "text"
},
"mpstat": {
"type": "text"
},
"oc": {
"type": "text"
},
"perf": {
"type": "text"
},
"pidstat": {
"type": "text"
},
"pprof": {
"type": "text"
},
"proc-interrupts": {
"type": "text"
},
"proc-vmstat": {
"type": "text"
},
"prometheus-metrics": {
"type": "text"
},
"sar": {
"type": "text"
},
"turbostat": {
"type": "text"
},
"vmstat": {
"type": "text"
},
}
},
},
},
"run": {
"properties": {
"config": {
"type": "keyword"
},
"controller": {
"type": "keyword"
},
"date": {
"type": "date"
},
"end": {
"type": "date"
},
"id": {
"type": "keyword"
},
"iterations": {
"type": "text"
},
"name": {
"type": "keyword"
},
"script": {
"type": "keyword"
},
"start": {
"type": "date"
},
"toolsgroup": {
"type": "keyword"
},
"user": {
"type": "keyword"
},
}
},
"sosreports": {
"type": "nested",
"properties": {
"hostname-f": {
"type": "keyword"
},
"hostname-s": {
"type": "keyword"
},
"inet": {
"type": "nested",
"properties": {
"ifname": {
"type": "keyword"
},
"ipaddr": {
"type": "ip"
},
},
},
"inet6": {
"type": "nested",
"properties": {
"ifname": {
"type": "keyword"
},
"ipaddr": {
"type": "keyword"
},
},
},
"md5": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"sosreport-error": {
"type": "text"
},
},
},
},
},
version=5,
)
elif name == "result":
return Template(
name="result",
idxname="result-data",
template_name="unit-test.v5.result-data",
file="run.json",
template_pattern="unit-test.v5.result-data.*",
index_template="unit-test.v5.result-data.{year}-{month}",
settings={"none": False},
mappings={
"_meta": {
"version": "5"
},
"date_detection": "false",
"properties": {
"@timestamp": {
"type": "date",
"format": "dateOptionalTime"
},
"@timestamp_original": {
"type": "keyword",
"index": "false"
},
"result_data_sample_parent": {
"type": "keyword"
},
"run": {
"properties": {
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
}
},
"iteration": {
"properties": {
"name": {
"type": "keyword"
},
"number": {
"type": "long"
},
}
},
"sample": {
"properties": {
"@idx": {
"type": "long"
},
"name": {
"type": "keyword"
},
"measurement_type": {
"type": "keyword"
},
"measurement_idx": {
"type": "long"
},
"measurement_title": {
"type": "text"
},
"uid": {
"type": "keyword"
},
}
},
"result": {
"properties": {
"@idx": {
"type": "long"
},
"read_or_write": {
"type": "long"
},
"value": {
"type": "double"
},
}
},
},
},
version=5,
)
else:
return None
def test_find_none(self, fake_mtime, db_session):
""" Test expected failure when we try to find a template that
does not exist.
"""
with pytest.raises(TemplateNotFound):
Template.find(name="data")
def __init__(self, config: PbenchServerConfig, logger: Logger):
super().__init__(config, logger, Schema())
template = Template.find("run")
self.template_name = template.template_name + "."
self.logger.info("month index key is {}", self.template_name)
def resolve(self):
"""
Check the on-disk template mapping files' modification dates against
the templates DB. Only if the template's base index name doesn't exist
in the DB, or if the DB object is older than the on-disk template do
we load the mapping files from disk.
"""
try:
template = Template.find(self.name)
# If we found a match that's not older, use it
if template.mtime >= self.modified:
self.update(template)
return
except TemplateNotFound:
template = None
# We didn't return, so the DB template is missing or older than the
# on-disk JSON. We now need to fully resolve the mapping files. If we
# found an older DB object, we'll update it with the new data;
# otherwise we'll create a new Template object.
self.load()
if not template:
template = Template(
name=self.name,
idxname=self.idxname,
template_name=self.template_name,
file=str(self.mappings.file),
template_pattern=self.index_pattern,
index_template=self.index_template,
settings=self.settings.json,
mappings=self.mappings.json,
mtime=self.modified,
version=self.version,
)
template.add()
else:
template.version = self.version
template.mappings = self.mappings.json
template.settings = self.settings.json
template.mtime = self.modified
template.update()