def kibana_url(self, function_name):
now = datetime.now(timezone.utc)
filters = [{
"query": {
"match_phrase": match_phrase
}
} for match_phrase in [
{
"function_name": function_name
},
{
"level": "error"
},
]]
time = {
key: time.isoformat().replace("+00:00", "Z")
for key, time in [
("from", now - timedelta(minutes=15)),
("to", now + timedelta(minutes=5)),
]
}
return "{}/discover#/?_a={}&_g={}".format(
os.environ.get("KIBANA_BASE_URL"),
prison.dumps({"filters": filters}),
prison.dumps({"time": time}),
)
def test_external_metadata_by_name_from_sqla_inspector(self):
self.login(username="******")
example_database = get_example_database()
with create_test_table_context(example_database):
params = prison.dumps(
{
"datasource_type": "table",
"database_name": example_database.database_name,
"table_name": "test_table",
"schema_name": get_example_default_schema(),
}
)
url = f"/datasource/external_metadata_by_name/?q={params}"
resp = self.get_json_resp(url)
col_names = {o.get("name") for o in resp}
self.assertEqual(col_names, {"first", "second"})
# No databases found
params = prison.dumps(
{
"datasource_type": "table",
"database_name": "foo",
"table_name": "bar",
}
)
url = f"/datasource/external_metadata_by_name/?q={params}"
resp = self.client.get(url)
self.assertEqual(resp.status_code, DatasetNotFoundError.status)
self.assertEqual(
json.loads(resp.data.decode("utf-8")).get("error"),
DatasetNotFoundError.message,
)
# No table found
params = prison.dumps(
{
"datasource_type": "table",
"database_name": example_database.database_name,
"table_name": "fooooooooobarrrrrr",
}
)
url = f"/datasource/external_metadata_by_name/?q={params}"
resp = self.client.get(url)
self.assertEqual(resp.status_code, DatasetNotFoundError.status)
self.assertEqual(
json.loads(resp.data.decode("utf-8")).get("error"),
DatasetNotFoundError.message,
)
# invalid query params
params = prison.dumps(
{
"datasource_type": "table",
}
)
url = f"/datasource/external_metadata_by_name/?q={params}"
resp = self.get_json_resp(url)
self.assertIn("error", resp)
def test_get_list_page(self):
"""
REST Api: Test get list page params
"""
page_size = 5
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
# test page zero
arguments = {
"page_size": page_size,
"page": 0,
"order_column": "field_integer",
"order_direction": "asc",
}
uri = "api/v1/model1api/?{}={}".format(API_URI_RIS_KEY,
prison.dumps(arguments))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(
data[API_RESULT_RES_KEY][0],
{
"field_date": None,
"field_float": 0.0,
"field_integer": 0,
"field_string": "test0",
},
)
eq_(rv.status_code, 200)
eq_(len(data[API_RESULT_RES_KEY]), page_size)
# test page one
arguments = {
"page_size": page_size,
"page": 1,
"order_column": "field_integer",
"order_direction": "asc",
}
uri = "api/v1/model1api/?{}={}".format(API_URI_RIS_KEY,
prison.dumps(arguments))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(
data[API_RESULT_RES_KEY][0],
{
"field_date": None,
"field_float": float(page_size),
"field_integer": page_size,
"field_string": "test{}".format(page_size),
},
)
eq_(rv.status_code, 200)
eq_(len(data[API_RESULT_RES_KEY]), page_size)
def create_tsvb(job, args):
"""
Create a Kibana TSVB Visualization based on args from reqparse
"""
assert args['type'] in Kibana.TSVB
vis_state = getattr(
Kibana,
'_create_{}_visualization'.format(args['type'].lower()))(job, args)
# additional global filter on data if provided.
if 'query' in args and args['query']:
vis_state['params']['filter']['query'] += \
' and ({})'.format(args['query'])
# rison-ify and replace
vis_state = Kibana._regex_process(prison.dumps(vis_state),
Kibana.RISON_REPLACEMENT)
start_time, end_time = Kibana._parse_start_end_time(args)
# a single-item list
return [
Kibana.BASIC_URL.format(type='metrics',
kbn_addr=Kibana.KIBANA_ADDRESS,
start_time=start_time,
end_time=end_time,
vis_state=vis_state)
]
def test_dict(self):
self.assertEqual('()', prison.dumps({}))
self.assertEqual('(a:0,b:1)', prison.dumps({'a': 0, 'b': 1}))
self.assertEqual("(a:0,b:foo,c:'23skidoo')",
prison.dumps({
'a': 0,
'c': '23skidoo',
'b': 'foo'
}))
self.assertEqual(
'(id:!n,type:/common/document)',
prison.dumps({
'type': '/common/document',
'id': None
}))
self.assertEqual("(a:0)", prison.dumps({'a': 0}))
def test_get_list_base_order(self):
"""
REST Api: Test get list with base order
"""
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
# test string order asc
rv = self.auth_client_get(client, token, "api/v1/model1apiorder/")
data = json.loads(rv.data.decode("utf-8"))
eq_(
data[API_RESULT_RES_KEY][0],
{
"field_date": None,
"field_float": float(MODEL1_DATA_SIZE - 1),
"field_integer": MODEL1_DATA_SIZE - 1,
"field_string": "test{}".format(MODEL1_DATA_SIZE - 1),
},
)
# Test override
arguments = {"order_column": "field_integer", "order_direction": "asc"}
uri = "api/v1/model1apiorder/?{}={}".format(API_URI_RIS_KEY,
prison.dumps(arguments))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(
data[API_RESULT_RES_KEY][0],
{
"field_date": None,
"field_float": 0.0,
"field_integer": 0,
"field_string": "test0",
},
)
def test_api_database(self):
self.login("admin")
self.create_fake_db()
arguments = {
"keys": [],
"filters": [{
"col": "expose_in_sqllab",
"opr": "eq",
"value": True
}],
"order_column": "database_name",
"order_direction": "asc",
"page": 0,
"page_size": -1,
}
url = "api/v1/database/?{}={}".format("q", prison.dumps(arguments))
self.assertEqual(
{"examples", "fake_db_100"},
{
r.get("database_name")
for r in self.get_json_resp(url)["result"]
},
)
self.delete_fake_db()
def test_info_select_meta_data(self):
"""
REST Api: Test info select meta data
"""
# select meta for add fields
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
selectable_keys = [
API_ADD_COLUMNS_RIS_KEY,
API_EDIT_COLUMNS_RIS_KEY,
API_PERMISSIONS_RIS_KEY,
API_FILTERS_RIS_KEY,
API_ADD_TITLE_RIS_KEY,
API_EDIT_TITLE_RIS_KEY,
]
for selectable_key in selectable_keys:
arguments = {API_SELECT_KEYS_RIS_KEY: [selectable_key]}
uri = "api/v1/model1api/_info?{}={}".format(
API_URI_RIS_KEY, prison.dumps(arguments))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(len(data.keys()), 1)
# We assume that rison meta key equals result meta key
assert selectable_key in data
def test_get_list_filters(self):
"""
REST Api: Test get list filter params
"""
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
filter_value = 5
# test string order asc
arguments = {
API_FILTERS_RIS_KEY: [
{"col": "field_integer", "opr": "gt", "value": filter_value}
],
"order_column": "field_integer",
"order_direction": "asc",
}
uri = "api/v1/model1api/?{}={}".format(API_URI_RIS_KEY, prison.dumps(arguments))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(
data[API_RESULT_RES_KEY][0],
{
"field_date": None,
"field_float": float(filter_value + 1),
"field_integer": filter_value + 1,
"field_string": "test{}".format(filter_value + 1),
},
)
eq_(rv.status_code, 200)
def create_tsvb(job, args):
"""
Create a Kibana TSVB Visualization based on args from reqparse
"""
assert args['type'] in Kibana.TSVB
if args['type'] == 'Rate':
vis_state = Kibana._create_rate_visualization(job, args)
elif args['type'] == 'Ratio':
vis_state = Kibana._create_ratio_visualization(job, args)
else:
assert args['type'] == 'Numeric'
vis_state = Kibana._create_numeric_visualization(job, args)
# additional global filter on data if provided.
if 'query' in args and args['query']:
vis_state['params']['filter']['query'] += \
' and ({})'.format(args['query'])
# rison-ify and replace
vis_state = Kibana._regex_process(prison.dumps(vis_state),
Kibana.RISON_REPLACEMENT)
start_time, end_time = Kibana._parse_start_end_time(args)
# a single-item list
return [
os.path.join(
Envs.KIBANA_ADDRESS,
Kibana.BASIC_QUERY.format(type='metrics',
start_time=start_time,
end_time=end_time,
vis_state=vis_state))
]
def test_get_filter_related_owners(self):
"""
Dashboard API: Test dashboard get filter related owners
"""
self.login(username="******")
argument = {"filter": "a"}
uri = "api/v1/dashboard/related/owners?{}={}".format(
"q", prison.dumps(argument))
rv = self.client.get(uri)
self.assertEqual(rv.status_code, 200)
response = json.loads(rv.data.decode("utf-8"))
expected_response = {
"count":
2,
"result": [
{
"text": "admin user",
"value": 1
},
{
"text": "alpha user",
"value": 5
},
],
}
self.assertEqual(response, expected_response)
def test_external_metadata_by_name_for_virtual_table(self):
self.login(username="******")
session = db.session
table = SqlaTable(
table_name="dummy_sql_table",
database=get_example_database(),
schema=get_example_default_schema(),
sql="select 123 as intcol, 'abc' as strcol",
)
session.add(table)
session.commit()
tbl = self.get_table(name="dummy_sql_table")
params = prison.dumps(
{
"datasource_type": "table",
"database_name": tbl.database.database_name,
"schema_name": tbl.schema,
"table_name": tbl.table_name,
}
)
url = f"/datasource/external_metadata_by_name/?q={params}"
resp = self.get_json_resp(url)
assert {o.get("name") for o in resp} == {"intcol", "strcol"}
session.delete(tbl)
session.commit()
def create_tsvb(job, args):
"""
Create a Kibana TSVB Visualization based on args from reqparse
"""
assert args['type'] in ('Rate', 'Ratio', 'Numeric')
vis_state = getattr(
KibanaUtils,
'_create_{}_visualization'.format(args['type'].lower()))(job, args)
# additional global filter on data if provided.
if args['query']:
vis_state['params']['filter']['query'] += \
' and ({})'.format(args['query'])
# rison-ify and replace
vis_state = KibanaUtils._regex_process(prison.dumps(vis_state),
KibanaUtils.RISON_REPLACEMENT)
start_time, end_time = KibanaUtils._parse_start_end_time(args)
return [
"{kbn_addr}/app/kibana#/visualize/create"
"?type=metrics&embed=true&"
"_g=(refreshInterval:(pause:!t,value:0),"
"time:(from:'{start_time}',to:'{end_time}'))&"
"_a=(filters:!(),linked:!f,"
"query:(language:kuery,query:''),uiState:(),"
"vis:{vis_state})".format(kbn_addr=KibanaUtils.KIBANA_ADDRESS,
start_time=start_time,
end_time=end_time,
vis_state=vis_state)
]
def kibana6_disover_global_state(from_time, to_time):
return prison.dumps({
'refreshInterval': {
'pause': True,
'value': 0
},
'time': {
'from': from_time,
'mode': 'absolute',
'to': to_time
}
})
def kibana7_disover_global_state(from_time, to_time):
return prison.dumps( {
'filters': [],
'refreshInterval': {
'pause': True,
'value': 0
},
'time': {
'from': from_time,
'to': to_time
}
} )
def kibana6_link(rule, starttime, endtime):
if len(rule["filter"]) == 1 and "query_string" in rule["filter"][0]:
query = rule["filter"][0]["query_string"]["query"]
return rule['kibana_url'] + "#/discover?_a=" + prison.dumps({
"time": {
"from": starttime,
"mode": "absolute",
"to": endtime
},
"query": {
"language": "lucene",
"query": query
},
})
else:
query = elastalert.ElastAlerter.get_query(rule["filter"], five=True)
filters = query["query"]["bool"]
return rule['kibana_url'] + "#/discover?_g=" + prison.dumps({
"filters": [{
"$state": {
"store": "globalState"
},
"bool": filters,
"meta": {
"alias": "filter",
"disabled": False,
"index": rule["index"],
"key": "bool",
"negate": False,
"type": "custom",
"value": json.dumps(filters),
},
}],
"time": {
"from": starttime,
"mode": "absolute",
"to": endtime
},
})
def test_external_metadata_by_name_for_physical_table(self):
self.login(username="******")
tbl = self.get_table(name="birth_names")
params = prison.dumps({
"datasource_type": "table",
"database_name": tbl.database.database_name,
"schema_name": tbl.schema,
"table_name": tbl.table_name,
})
url = f"/datasource/external_metadata_by_name/?q={params}"
resp = self.get_json_resp(url)
col_names = {o.get("name") for o in resp}
self.assertEqual(
col_names,
{"num_boys", "num", "gender", "name", "ds", "state", "num_girls"})
def test_get_dashboards_api_no_data_access(self):
"""
Dashboard API: Test get dashboards no data access
"""
admin = self.get_user("admin")
title = f"title{random_str()}"
create_dashboard_to_db(title, "slug1", owners=[admin])
self.login(username="******")
arguments = {
"filters": [{"col": "dashboard_title", "opr": "sw", "value": title[0:8]}]
}
uri = DASHBOARDS_API_URL_WITH_QUERY_FORMAT.format(prison.dumps(arguments))
rv = self.client.get(uri)
self.assert200(rv)
data = json.loads(rv.data.decode("utf-8"))
self.assertEqual(0, data["count"])
def discover_app_state(self, query, index, columns, alias, filters):
if not filters:
app_filters = []
for k, v in query.items():
app_filters.append({
'$state': {
'store': 'appState'
},
'meta': {
'alias': alias,
'disabled': False,
'index': index,
'key': k,
'negate': False,
'params': {
'query': v
},
'type': 'phrase'
},
'query': {
'match_phrase': {
k: v
}
}
})
else:
app_filters = [{
"$state": {
"store": "appState"
},
"meta": {
"alias": alias,
"disabled": False,
"index": index
},
"query": {
"bool": query
}
}]
return prison.dumps({
'columns': columns,
'filters': app_filters,
'index': index,
'interval': 'auto'
})
def test_get_list_max_page_size(self):
"""
REST Api: Test get list max page size config setting
"""
page_size = 100 # Max is globally set to MAX_PAGE_SIZE
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
# test page zero
arguments = {
"page_size": page_size,
"page": 0,
"order_column": "field_integer",
"order_direction": "asc",
}
uri = "api/v1/model1api/?{}={}".format(API_URI_RIS_KEY, prison.dumps(arguments))
print("URI {}".format(uri))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(len(data[API_RESULT_RES_KEY]), MAX_PAGE_SIZE)
def test_api_database(self):
self.login("admin")
arguments = {
"keys": [],
"filters": [{
"col": "expose_in_sqllab",
"opr": "eq",
"value": True
}],
"order_column": "database_name",
"order_direction": "asc",
"page": 0,
"page_size": -1,
}
expected_results = ["examples", "fake_db_100", "main"]
url = "api/v1/database/?{}={}".format("q", prison.dumps(arguments))
data = self.get_json_resp(url)
for i, expected_result in enumerate(expected_results):
self.assertEquals(expected_result,
data["result"][i]["database_name"])
def test_get_list_select_cols(self):
"""
REST Api: Test get list with selected columns
"""
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
argument = {
API_SELECT_COLUMNS_RIS_KEY: ["field_integer"],
"order_column": "field_integer",
"order_direction": "asc",
}
uri = "api/v1/model1api/?{}={}".format(API_URI_RIS_KEY, prison.dumps(argument))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(data[API_RESULT_RES_KEY][0], {"field_integer": 0})
eq_(data[API_LABEL_COLUMNS_RES_KEY], {"field_integer": "Field Integer"})
eq_(data[API_DESCRIPTION_COLUMNS_RES_KEY], {"field_integer": "Field Integer"})
eq_(data[API_LIST_COLUMNS_RES_KEY], ["field_integer"])
eq_(rv.status_code, 200)
def test_get_item_select_meta_data(self):
"""
REST Api: Test get item select meta data
"""
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
selectable_keys = [
API_DESCRIPTION_COLUMNS_RIS_KEY,
API_LABEL_COLUMNS_RIS_KEY,
API_SHOW_COLUMNS_RIS_KEY,
API_SHOW_TITLE_RIS_KEY,
]
for selectable_key in selectable_keys:
argument = {API_SELECT_KEYS_RIS_KEY: [selectable_key]}
uri = "api/v1/model1api/1?{}={}".format(API_URI_RIS_KEY,
prison.dumps(argument))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
eq_(len(data.keys()), 1 + 2) # always exist id, result
# We assume that rison meta key equals result meta key
assert selectable_key in data
def test_get_list_base_filters(self):
"""
REST Api: Test get list with base filters
"""
client = self.app.test_client()
token = self.login(client, USERNAME, PASSWORD)
arguments = {
"order_column": "field_integer",
"order_direction": "desc"
}
uri = "api/v1/model1apifiltered/?{}={}".format(API_URI_RIS_KEY,
prison.dumps(arguments))
rv = self.auth_client_get(client, token, uri)
data = json.loads(rv.data.decode("utf-8"))
expected_result = [{
"field_date": None,
"field_float": 3.0,
"field_integer": 3,
"field_string": "test3",
}]
eq_(data[API_RESULT_RES_KEY], expected_result)
def test_dumps(self):
pfile = open("test_data/example.prison")
data = pfile.read()
pfile.close()
data = prison.loads(data)
data = prison.dumps(data)
def risonify(data) -> str:
return rison_quote(prison.dumps(data)[1:-1])
def kibana_discover_app_state(index, columns, filters, query_keys, match):
app_filters = []
if filters:
# Remove nested query since the outer most query key will break Kibana 8.
new_filters = []
for filter in filters:
if 'query' in filter:
filter = filter['query']
new_filters.append(filter)
filters = new_filters
bool_filter = { 'must': filters }
app_filters.append( {
'$state': {
'store': 'appState'
},
'bool': bool_filter,
'meta': {
'alias': 'filter',
'disabled': False,
'index': index,
'key': 'bool',
'negate': False,
'type': 'custom',
'value': json.dumps(bool_filter, separators=(',', ':'))
},
} )
for query_key in query_keys:
query_value = lookup_es_key(match, query_key)
if query_value is None:
app_filters.append( {
'$state': {
'store': 'appState'
},
'exists': {
'field': query_key
},
'meta': {
'alias': None,
'disabled': False,
'index': index,
'key': query_key,
'negate': True,
'type': 'exists',
'value': 'exists'
}
} )
else:
app_filters.append( {
'$state': {
'store': 'appState'
},
'meta': {
'alias': None,
'disabled': False,
'index': index,
'key': query_key,
'negate': False,
'params': {
'query': query_value,
'type': 'phrase'
},
'type': 'phrase',
'value': str(query_value)
},
'query': {
'match': {
query_key: {
'query': query_value,
'type': 'phrase'
}
}
}
} )
return prison.dumps( {
'columns': columns,
'filters': app_filters,
'index': index,
'interval': 'auto'
} )
def get_live_crawls(config, projects_id, timestamp_range, limit=None):
headers['Authorization'] = 'Bearer {}'.format(config['api_key'])
offset = 0
if limit is None:
limit = 1000
filters = {
"and": [{
"field": ["status", "equals", "done"]
}, {
"field": ["created_at", "gte", timestamp_range['start']]
}, {
"field": ["created_at", "lt", timestamp_range['end']]
}, {
"field": ["project_id", "one_of", projects_id]
}]
}
try:
response = []
while offset is not None:
r = requests.get(
'https://app.oncrawl.com/api/v2/crawls?filters={}&offset={}&limit={}&sort=created_at:desc'
.format(prison.dumps(filters), offset, limit),
headers=headers)
r.raise_for_status()
items = r.json()
offset = int(items['meta']['offset']) + int(items['meta']['limit'])
for item in items['crawls']:
if (config['index'] == 'pages' and item['status']
== 'done') or (config['index'] == 'links'
and item['link_status'] == 'live'):
response.append({
'id': item['id'],
'config_name': item['crawl_config']['name'],
'project_id': item['project_id'],
'created_at': item['created_at'],
'ended_at': item['ended_at']
})
assert offset <= items['meta']['total']
except AssertionError:
offset = None
except requests.exceptions.HTTPError as e:
offset = None
response = {'error': build_human_error(r)}
except Exception as e:
response = {'error': repr(e)}
return response
def kibana_discover_app_state(index, columns, filters, query_keys, match):
app_filters = []
if filters:
bool_filter = {'must': filters}
app_filters.append({
'$state': {
'store': 'appState'
},
'bool': bool_filter,
'meta': {
'alias': 'filter',
'disabled': False,
'index': index,
'key': 'bool',
'negate': False,
'type': 'custom',
'value': json.dumps(bool_filter, separators=(',', ':'))
},
})
for query_key in query_keys:
query_value = dots_get(match, query_key)
if query_value is None:
app_filters.append({
'$state': {
'store': 'appState'
},
'exists': {
'field': query_key
},
'meta': {
'alias': None,
'disabled': False,
'index': index,
'key': query_key,
'negate': True,
'type': 'exists',
'value': 'exists'
}
})
else:
app_filters.append({
'$state': {
'store': 'appState'
},
'meta': {
'alias': None,
'disabled': False,
'index': index,
'key': query_key,
'negate': False,
'params': {
'query': query_value,
'type': 'phrase'
},
'type': 'phrase',
'value': str(query_value)
},
'query': {
'match': {
query_key: {
'query': query_value,
'type': 'phrase'
}
}
}
})
return prison.dumps({
'columns': columns,
'filters': app_filters,
'index': index,
'interval': 'auto'
})
def bulk_delete_dashboard_via_api(self, dashboard_ids):
uri = DASHBOARDS_API_URL_WITH_QUERY_FORMAT.format(
prison.dumps(dashboard_ids))
return self.delete_assert_metric(uri, "bulk_delete")