def looker_cleanup_models(client_properties, in_access_token, project_name = ""):
"""
Function returns
:param client_properties:
:param in_access_token:
:param project_name
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "GET_LOOKML_MODELS")
resp_code = get_response_code(r)
body = r.json()
explore_list = []
if resp_code == 200:
for model in body:
if not model["has_content"]:
debug('Deleting model {} '.format(model["name"]), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "DELETE_LOOKML_MODEL", model["name"])
resp_code = get_response_code(r)
if resp_code == 204:
pass
else:
body = r.json()
debug("Cannot delete model {} {}".format(model["name"], body["message"]), _WARNING)
else:
debug("Cannot fetch models {}".format(body["message"]), _WARNING)
return False
def looker_looks(client_properties, in_access_token, all_looks = False):
"""
:param client_properties:
:param in_access_token:
:param space_name:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
namespaces = client_properties["space_remap"]
r = run_looker_restapi(client_properties, in_access_token, "GET_LOOKS")
resp_code = get_response_code(r)
body = r.json()
looks = []
if resp_code == 200:
if all_looks:
# we just return all looks if requested w/o body
debug("Return ALL non-deleted server looks", _INFO)
return [el for el in body if not el["deleted"]]
for look in body:
if len(namespaces) == 0 or "" in namespaces.keys() or look["space"]["name"] in namespaces.keys():
if look["deleted"]:
debug("Look deleted, skipping: {}: {} ".format(look["space"]["name"], look["title"]), _INFO)
else:
debug("Found Look {}: {}".format(look["space"]["name"], look["title"]), _INFO)
look["look_json"] = looker_look(client_properties, in_access_token, look["id"])
looks.append(look)
return looks
else:
debug("Cannot get Looks: {}".format(body["message"]), _WARNING)
def looker_run_query(client_properties, in_access_token, query, output_format, query_title=""):
"""
:param client_properties:
:param in_access_token:
:param query:
:param output_format:
:param query_title:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
debug("Running Query {}:{} ({})".format(query["model"], query["view"], query_title), _INFO)
global _GLOBAL_SUMMARY
r = run_looker_restapi(client_properties, in_access_token, "RUN_INLINE_QUERY", output_format, in_payload=query)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
log_msg = "Output {} rows".format(len(body))
_GLOBAL_SUMMARY.append(log_msg)
debug(log_msg, _INFO)
if len(body) >0 and "looker_error" in body[0].keys():
for looker_err in body:
err = "SQL ERROR REPORTED {}".format(looker_err["looker_error"])
_GLOBAL_SUMMARY.append("ERROR: running query: {}".format(err))
debug(err, _WARNING)
return False
else:
debug("Cannot run query {}".format(body["message"]), _WARNING)
return False
return body
def looker_get_lookml_models(client_properties, in_access_token, project_name = ""):
"""
Function returns
:param client_properties:
:param in_access_token:
:param project_name
:return: dictionary of models and explores
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "GET_LOOKML_MODELS")
resp_code = get_response_code(r)
body = r.json()
explore_list = []
if resp_code == 200:
for exp in body:
if project_name == "" or exp["project_name"] == project_name:
new_model = {
"project_name": exp["project_name"],
"name": exp["name"],
"explores": [nm["name"] for nm in exp["explores"] if client_properties.get("test_hidden_explores","True") == "True" or not nm["hidden"]]
}
explore_list.append(new_model)
debug("Found model: {}".format(new_model), _INFO)
else:
debug("Cannot fetch models/explores {}".format(body["message"]), _WARNING)
return False
return explore_list
def looker_dashboards(client_properties, in_access_token, fetch_dashboards = True):
"""
:param client_properties:
:param in_access_token:
:param fetch_dashboards:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
namespaces = client_properties["space_remap"]
r = run_looker_restapi(client_properties, in_access_token, "GET_DASHBOARDS")
resp_code = get_response_code(r)
body = r.json()
dashboards = []
if resp_code == 200:
if not fetch_dashboards:
return body
for dashboard in body:
if dashboard.get("space", "") and \
(len(namespaces) == 0 or "" in namespaces.keys() or dashboard["space"]["name"] in namespaces.keys()):
debug("Found Dashboard {}: {}".format(dashboard["space"]["name"], dashboard["title"]), _INFO)
dashboard["dashboard_json"] = looker_get_dashboard(client_properties, in_access_token, dashboard["id"])
dashboards.append(dashboard)
return dashboards
else:
debug("Cannot get Dashboards: {}".format(body["message"]), _WARNING)
def looker_create_query(client_properties, in_access_token, query):
"""
:param client_properties:
:param in_access_token:
:param query:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
query_id = query["id"]
new_query = query.copy()
for key in _QUERY_KEYS_TO_REMOVE:
del new_query[key]
#print(json.dumps(new_query, indent=4, separators=(',', ': ')))
new_query["model"] = replace_model(client_properties, new_query["model"])
r = run_looker_restapi(client_properties, in_access_token, "CREATE_QUERY", in_payload=new_query)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
debug("Created Query (original id: {}) : {}".format(query_id, body["id"]), _INFO)
return body["id"]
else:
debug("Cannot create Query {}".format(body["message"]), _WARNING)
return
def looker_create_dashboard_filters(client_properties, in_access_token, dashboard_id, filter_list):
"""
:param client_properties:
:param in_access_token:
:param dashboard_id:
:param filter_list:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
for dashboard_filter in filter_list:
debug("Creating filter {}".format(dashboard_filter["name"]), _INFO)
new_filter = dashboard_filter.copy()
del new_filter["id"]
new_filter["dashboard_id"] = dashboard_id
new_filter["model"] = replace_model(client_properties, new_filter["model"])
r = run_looker_restapi(client_properties, in_access_token, "CREATE_DASHBOARD_FILTER", in_payload=new_filter)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
debug("Created Dashboard Filter {} (original id: {}) : {}".format(new_filter["name"], dashboard_filter["id"], body["id"]), _INFO)
filter_id = body["id"]
else:
debug("Cannot create filter {}".format(body["message"]), _WARNING)
return False
return True
def looker_who_am_I(client_properties, in_access_token):
"""
:param client_properties:
:param in_access_token:
:return user_id:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "WHO_AM_I")
resp_code = get_response_code(r)
body = r.json()
space_id = None
if resp_code == 200:
debug("You are {} {} ({}) user_id: {}".format(body["first_name"], body["last_name"], body["email"], body["id"]), _INFO)
return body["id"]
else:
debug("Cannot determine current user name: {}".format(body["message"]), _WARNING)
def looker_spaces(client_properties, in_access_token):
"""
:param client_properties:
:param in_access_token:
:param space_name:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "GET_SPACES")
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
return {el["name"]:el["id"] for el in body}
else:
debug("Cannot get Spaces: {}".format(body["message"]), _WARNING)
def looker_get_dashboard(client_properties, in_access_token, dashboard_id):
"""
:param client_properties:
:param in_access_token:
:param dashboard_id:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "GET_DASHBOARD", dashboard_id)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
return body
else:
debug("Cannot get Dashboard: {}".format(body["message"]), _WARNING)
def looker_create_look(client_properties, in_access_token, look, space_list):
"""
:param client_properties:
:param in_access_token:
:param look:
:param space_list:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
# Fetching namespace of the original look
namespace = look["space"]["name"]
(namespace_id, new_namespace) = get_namespace(client_properties, namespace, space_list)
title = "{}{}".format(client_properties.get("name_prefix", ""), look["title"])
debug("Creating Look {}, space: {} ({}) ".format(title, new_namespace, namespace_id), _INFO)
if not namespace_id:
debug("Cannot find space for Look {}, space: {} skipping creation. ".format(title, namespace), _WARNING)
return
query_id = looker_create_query(client_properties, in_access_token, look["query"])
if not query_id:
return
new_look = {
"query_id": query_id,
"space_id": namespace_id,
"title": title
}
copy_dict(["can", "description", "is_run_on_load"], look, new_look)
r = run_looker_restapi(client_properties, in_access_token, "CREATE_LOOK", in_payload=new_look)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
debug("Created Look (original id: {}) : {}".format(look["id"], body["id"]), _INFO)
return body["id"]
else:
debug("Cannot create look {}".format(body["message"]), _WARNING)
return
def looker_get_explore(client_properties, in_access_token, model_name, exp_name):
"""
:param client_properties:
:param in_access_token:
:param model_name:
:param exp_name:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "GET_EXPLORE", model_name, exp_name)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
debug("Fetched Explore {}".format(body["id"]), _INFO)
return body
else:
debug("Cannot get explore {}:{} {}".format(model_name, exp_name, body["message"]), _WARNING)
return
def looker_delete_dashboard(client_properties, in_access_token, dashboard_id):
"""
:param client_properties:
:param in_access_token:
:param dashboard_id:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
debug("Deleting incomplete dashboard {}".format(dashboard_id), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "DELETE_DASHBOARD", dashboard_id)
resp_code = get_response_code(r)
if resp_code == 204:
pass
else:
body = r.json()
debug("Cannot delete dashboard {} {}".format(dashboard_id, body["message"]), _WARNING)
return False
return True
def looker_delete_dashboard_layouts(client_properties, in_access_token, new_layouts):
"""
:param client_properties:
:param in_access_token:
:param new_layouts:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
for dashboard_layout in new_layouts:
debug("Deleting old layout {}".format(dashboard_layout["id"]), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "DELETE_DASHBOARD_LAYOUT", dashboard_layout["id"])
resp_code = get_response_code(r)
if resp_code == 204:
pass
else:
body = r.json()
debug("Cannot delete layout {} {}".format(dashboard_layout["id"], body["message"]), _WARNING)
return False
return True
def looker_get_space_id(client_properties, in_access_token, space_name):
"""
:param client_properties:
:param in_access_token:
:param space_name:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
r = run_looker_restapi(client_properties, in_access_token, "FIND_SPACE", space_name)
resp_code = get_response_code(r)
body = r.json()
space_id = None
if resp_code == 200:
if body:
space_id = body[0]["id"]
debug("Found space: {} ({})".format(space_name, space_id), _INFO)
else:
debug("Space {} not found".format(space_name, space_id), _WARNING)
return space_id
else:
debug("Cannot find space: {}".format(body["message"]), _WARNING)
def looker_create_dashboard_layouts(client_properties, in_access_token, dashboard_id, element_list, new_layouts, layout_list):
"""
:param client_properties:
:param in_access_token:
:param dashboard_id:
:param element_list:
:param new_layouts:
:param layout_list:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
for dashboard_layout in layout_list:
debug("Creating layout {}".format(dashboard_layout["id"]), _INFO)
new_layout = dashboard_layout.copy()
layout_component = dashboard_layout["dashboard_layout_components"]
del new_layout["id"]
del new_layout["dashboard_layout_components"]
new_layout["dashboard_id"] = dashboard_id
#print(json.dumps(new_layout, indent=4, separators=(',', ': ')))
r = run_looker_restapi(client_properties, in_access_token, "CREATE_DASHBOARD_LAYOUT", in_payload=new_layout)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
layout_id = body["id"]
debug("Created Dashboard Layout {} (original id: {}) ".format(layout_id, dashboard_layout["id"]), _INFO)
#print(json.dumps(body["dashboard_layout_components"], indent=4, separators=(',', ': ')))
#Process components
for component in body["dashboard_layout_components"]:
old_element_id = element_list [component["dashboard_element_id"]]
new_component = None
for old_component in layout_component:
if old_component["dashboard_element_id"] == old_element_id:
new_component = old_component.copy()
if not new_component:
debug("Cannot locate original component {}".format(old_element_id), _WARNING)
return False
new_component["dashboard_layout_id"] = layout_id
new_component["id"] = component["id"]
new_component["dashboard_element_id"] = component["dashboard_element_id"]
#print(json.dumps(new_component, indent=4, separators=(',', ': ')))
r = run_looker_restapi(client_properties, in_access_token, "UPDATE_DASHBOARD_LAYOUT_COMPONENT",
component["id"], in_payload=new_component)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
debug("Updated Dashboard Layout Component {} ".format(new_component["id"]), _INFO)
else:
debug("Cannot update layout component {}".format(body["message"]), _WARNING)
return False
else:
debug("Cannot create layout {}".format(body["message"]), _WARNING)
return False
# Now we delete default layouts created automatically.
if not looker_delete_dashboard_layouts(client_properties, in_access_token, new_layouts):
debug("Cannot delete Default Layouts", _WARNING)
return False
return True
def looker_create_dashboard_elements(client_properties, in_access_token, server_looks, dashboard_id, namespace_id, element_list):
"""
:param client_properties:
:param in_access_token:
:param dashboard_id:
:param namespace_id:
:param element_list:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
new_element_list = {}
for dashboard_element in element_list:
debug("Creating element {}".format(dashboard_element["id"]), _INFO)
new_dashboard_element = dashboard_element.copy()
if new_dashboard_element["result_maker_id"]:
del new_dashboard_element["result_maker"]["id"]
for el in ["id", "look", "result_maker_id"]:
del new_dashboard_element[el]
new_dashboard_element["dashboard_id"] = dashboard_id
if new_dashboard_element["look_id"]:
# We need to find LOOK_ID
new_look_id = None
look_name = dashboard_element["look"]["title"]
new_look_name = title = "{}{}".format(client_properties.get("name_prefix", ""), look_name)
# First we try to see if we have a look with remapped name in the new space
for look in server_looks:
if look["space_id"] == namespace_id and look["title"] == new_look_name:
new_look_id = look["id"]
debug("Found look in the new space {} ({}) namespace {}".format(new_look_name, new_look_id, namespace_id), _INFO)
break
if not new_look_id:
debug("Cannot find look with the provided name {} in the namespace {}".format(new_look_name, namespace_id), _WARNING)
return {}
new_dashboard_element["look_id"] = new_look_id
elif new_dashboard_element["query_id"]:
debug("Element with query {}, creating...".format(new_dashboard_element["query_id"]), _INFO)
new_query_id = looker_create_query(client_properties, in_access_token, new_dashboard_element["query"])
if not new_query_id:
debug("Cannot create query", _WARNING)
return {}
del new_dashboard_element["query"]
new_dashboard_element["query_id"] = new_query_id
r = run_looker_restapi(client_properties, in_access_token,
"CREATE_DASHBOARD_ELEMENT", in_payload=new_dashboard_element)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
new_element_id = body["id"]
debug("Created Dashboard Element {} (original id: {})".format(new_element_id, dashboard_element["id"]), _INFO)
new_element_list[new_element_id] = dashboard_element["id"]
else:
debug("Cannot create element {}".format(body["message"]), _WARNING)
return {}
return new_element_list
def looker_create_dashboard(client_properties, in_access_token, server_looks, dashboard, space_list):
"""
:param client_properties:
:param in_access_token:
:param server_looks:
:param dashboard:
:param space_list:
:return:
"""
debug("*** Function call - {}".format(sys._getframe().f_code.co_name), _INFO)
# Fetching namespace of the original dashboard
namespace = dashboard["space"]["name"]
(namespace_id, new_namespace) = get_namespace(client_properties, namespace, space_list)
title = "{}{}".format(client_properties.get("name_prefix", ""), dashboard["title"])
debug("Creating Dashboard {}, space: {} ({}) ".format(title, new_namespace, namespace_id), _INFO)
if not namespace_id:
debug("Cannot find space for Dashboard {}, space: {} skipping creation. ".format(title, namespace), _WARNING)
return
dashboard_id = None
error_flag = False
new_dashboard = {
"space_id": namespace_id,
"title": title
}
attr_list_to_copy =["hidden",
"refresh_interval",
"load_configuration",
"background_color",
"show_title",
"title_color",
"show_filters_bar",
"tile_background_color",
"text_tile_text_color",
"query_timezone",
"can"]
copy_dict(attr_list_to_copy, dashboard, new_dashboard)
r = run_looker_restapi(client_properties, in_access_token, "CREATE_DASHBOARD", in_payload=new_dashboard)
resp_code = get_response_code(r)
body = r.json()
if resp_code == 200:
debug("Created Dashboard (original id: {}) : {}".format(dashboard["id"], body["id"]), _INFO)
dashboard_id = body["id"]
new_layouts = body["dashboard_layouts"]
else:
debug("Cannot create Dashboard {}".format(body["message"]), _WARNING)
error_flag = True
if "dashboard_filters" in dashboard and not error_flag:
if not looker_create_dashboard_filters(client_properties, in_access_token, dashboard_id, dashboard["dashboard_filters"]):
debug("Cannot Create Dashboard Filters", _WARNING)
error_flag = True
if "dashboard_elements" in dashboard and not error_flag:
element_list = looker_create_dashboard_elements(client_properties, in_access_token, server_looks, dashboard_id,
namespace_id, dashboard["dashboard_elements"])
if not element_list:
debug("Cannot Create Dashboard Elements", _WARNING)
error_flag = True
if "dashboard_layouts" in dashboard and not error_flag:
if not looker_create_dashboard_layouts(client_properties, in_access_token, dashboard_id,
element_list, new_layouts, dashboard["dashboard_layouts"]):
debug("Cannot Create Dashboard Layouts", _WARNING)
error_flag = True
# Check if the flow completed. If not, try to cleanup
if error_flag and dashboard_id:
looker_delete_dashboard(client_properties, in_access_token, dashboard_id)
return False
return dashboard_id